From 160054e33c386c3da391eeb41ccf9227b6c5b46d Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 27 Nov 2018 06:01:12 -0600 Subject: [PATCH 01/41] Use only safe axis types for Broadcast.combine_axes #30074 used the wrong notion of consistency since `OneTo(1)` is consistent (wrt broadcasting) with any range, but `OneTo` cannot handle `Slice(-1:1)`. (cherry picked from commit 1884cb4e3b17e24a6e9f5ad0bd653cfe7cbf070f) --- base/broadcast.jl | 11 ++++------- test/offsetarray.jl | 4 ++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 712aaa5947b66..db85efd85c281 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -436,19 +436,16 @@ end _bcs1(a::Integer, b::Integer) = a == 1 ? b : (b == 1 ? a : (a == b ? a : throw(DimensionMismatch("arrays could not be broadcast to a common size")))) _bcs1(a::Integer, b) = a == 1 ? b : (first(b) == 1 && last(b) == a ? b : throw(DimensionMismatch("arrays could not be broadcast to a common size"))) _bcs1(a, b::Integer) = _bcs1(b, a) -_bcs1(a, b) = _bcsm(b, a) ? _sametype(b, a) : (_bcsm(a, b) ? _sametype(a, b) : throw(DimensionMismatch("arrays could not be broadcast to a common size"))) +_bcs1(a, b) = _bcsm(b, a) ? axistype(b, a) : (_bcsm(a, b) ? axistype(a, b) : throw(DimensionMismatch("arrays could not be broadcast to a common size"))) # _bcsm tests whether the second index is consistent with the first _bcsm(a, b) = a == b || length(b) == 1 _bcsm(a, b::Number) = b == 1 _bcsm(a::Number, b::Number) = a == b || b == 1 # Ensure inferrability when dealing with axes of different AbstractUnitRange types # (We may not want to define general promotion rules between, say, OneTo and Slice, but if -# we get here we know the axes are at least consistent) -_sametype(a::T, b::T) where T = a -_sametype(a::OneTo, b::OneTo) = OneTo{Int}(a) -_sametype(a::OneTo, b) = OneTo{Int}(a) -_sametype(a, b::OneTo) = OneTo{Int}(a) -_sametype(a, b) = UnitRange{Int}(a) +# we get here we know the axes are at least consistent for the purposes of broadcasting) +axistype(a::T, b::T) where T = a +axistype(a, b) = UnitRange{Int}(a) ## Check that all arguments are broadcast compatible with shape # comparing one input against a shape diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 5aa58853179eb..e3104c015b6e9 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -470,6 +470,10 @@ A = OffsetArray(view(rand(4,4), 1:4, 4:-1:1), (-3,5)) a = [1] b = OffsetArray(a, (0,)) @test @inferred(a .+ b) == [2] +a = OffsetArray([1, -2, 1], (-2,)) +@test a .* a' == OffsetArray([ 1 -2 1; + -2 4 -2; + 1 -2 1], (-2,-2)) end # let From d9a35922540d428ad7ba36f71fc97787c0b81711 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 7 Dec 2018 21:43:43 -0500 Subject: [PATCH 02/41] fix #30303, escaping $ when showing Symbols (#30304) * fix #30303, escaping $ when showing Symbols * use repr instead of escape_string (cherry picked from commit f0b9499f5f311567ba4fda1bf14355009d592ff3) --- base/show.jl | 2 +- test/show.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index d4e22a9b1d832..0b37ca094ae3a 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1031,7 +1031,7 @@ function show_unquoted_quote_expr(io::IO, @nospecialize(value), indent::Int, pre print(io, ":") print(io, value) else - print(io, "Symbol(\"", escape_string(s), "\")") + print(io, "Symbol(", repr(s), ")") end else if isa(value,Expr) && value.head === :block diff --git a/test/show.jl b/test/show.jl index 870156744751d..30f1ccffb02e2 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1402,3 +1402,6 @@ end replstrcolor(x) = sprint((io, x) -> show(IOContext(io, :limit => true, :color => true), MIME("text/plain"), x), x) @test occursin("\e[", replstrcolor(`curl abc`)) + +# issue #30303 +@test repr(Symbol("a\$")) == "Symbol(\"a\\\$\")" From 7df523a581886e888d49d9e1b15e8bbe2cd919fd Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Sun, 9 Dec 2018 13:57:44 -0800 Subject: [PATCH 03/41] Use `JL_AArch64_crc` instead of `HWCAP_CRC32` (#30324) Closes https://github.com/JuliaLang/julia/issues/26458 (cherry picked from commit bd21aa75eb4c0e56ef870f6bffea3281c724cbb1) --- src/crc32c.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crc32c.c b/src/crc32c.c index f9100a22abe93..9ad1991d3c804 100644 --- a/src/crc32c.c +++ b/src/crc32c.c @@ -43,6 +43,7 @@ #include "julia.h" #include "julia_internal.h" +#include "processor.h" #ifdef _CPU_AARCH64_ # include @@ -333,7 +334,7 @@ JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const char *buf, size_t len) # else static crc32c_func_t crc32c_dispatch(unsigned long hwcap) { - if (hwcap & HWCAP_CRC32) + if (hwcap & (1 << JL_AArch64_crc)) return crc32c_armv8; return jl_crc32c_sw; } From 52bf7b923db2496c64cf145c57ae87be9f65ad74 Mon Sep 17 00:00:00 2001 From: Andy Ferris Date: Tue, 11 Dec 2018 12:06:55 +1000 Subject: [PATCH 04/41] `@inbounds` annotations for filter (#30156) (cherry picked from commit 58f9bf7042c0e0ad8a50b0914a5aff76613962b6) --- base/array.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index a6364188b8977..a10678d44b1c6 100644 --- a/base/array.jl +++ b/base/array.jl @@ -2337,7 +2337,7 @@ function filter!(f, a::AbstractVector) for acurr in a if f(acurr) - a[i] = acurr + @inbounds a[i] = acurr y = iterate(idx, state) y === nothing && (i += 1; break) i, state = y From 2cadff30d6bf47b658b6b0802d0ad8023d45652e Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 11 Dec 2018 02:28:26 -0500 Subject: [PATCH 05/41] Expand and fix documentation of BitArray (#30340) Fixes #30337... and while I was there I added a bit more information about what BitArrays do and when you might run into them. (cherry picked from commit 0d620001c200e49e2882500c94b9a150124bf875) --- base/bitarray.jl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index afae953078d4d..6ad196a0ba77e 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -5,9 +5,16 @@ # notes: bits are stored in contiguous chunks # unused bits must always be set to 0 """ - BitArray{N} <: DenseArray{Bool, N} + BitArray{N} <: AbstractArray{Bool, N} -Space-efficient `N`-dimensional boolean array, which stores one bit per boolean value. +Space-efficient `N`-dimensional boolean array, using just one bit for each boolean value. + +`BitArray`s pack up to 64 values into every 8 bytes, resulting in an 8x space efficiency +over `Array{Bool, N}` and allowing some operations to work on 64 values at once. + +By default, Julia returns `BitArrays` from [broadcasting](@ref Broadcasting) operations +that generate boolean elements (including dotted-comparisons like `.==`) as well as from +the functions [`trues`](@ref) and [`falses`](@ref). """ mutable struct BitArray{N} <: AbstractArray{Bool, N} chunks::Vector{UInt64} From e7187f6d0fc8b935482851b5a73444ca27298537 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Tue, 11 Dec 2018 16:29:41 +0100 Subject: [PATCH 06/41] Update to Documenter 0.21 and prepare for PDF documentation builds. (#30339) (cherry picked from commit 217d330296debe0567bb07addabf66b00602e325) --- doc/Makefile | 3 ++- doc/Manifest.toml | 27 ++++++++++++++------------- doc/Project.toml | 1 + doc/make.jl | 17 +++++++++++++---- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 3d0773342fc1d..99e60ee665c1f 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -20,7 +20,8 @@ help: @echo "To fix outdated doctests, use 'make doctest=fix'" -DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) +DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ + texplatform=$(texplatform) UnicodeData.txt: $(JLDOWNLOAD) http://www.unicode.org/Public/9.0.0/ucd/UnicodeData.txt diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 609d04b817e2b..5cbd6d684b7ed 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -1,3 +1,5 @@ +# This file is machine-generated - editing it directly is not advised + [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -6,35 +8,34 @@ deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[Distributed]] -deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[DocStringExtensions]] deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "a016e0bfe98a748c4488e2248c2ef4c67d6fdd35" +git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.5.0" +version = "0.6.0" [[Documenter]] deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] -git-tree-sha1 = "9f2135e0e7ecb63f9c3ef73ea15a31d8cdb79bb7" +git-tree-sha1 = "a6db1c69925cdc53aafb38caec4446be26e0c617" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.20.0" +version = "0.21.0" + +[[DocumenterLaTeX]] +deps = ["Documenter", "Test"] +git-tree-sha1 = "653299370be20ff580bccd707dc9f360c0852d7f" +uuid = "cd674d7a-5f81-5cf3-af33-235ef1834b99" +version = "0.2.0" [[InteractiveUtils]] -deps = ["LinearAlgebra", "Markdown"] +deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[LibGit2]] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/doc/Project.toml b/doc/Project.toml index dfa65cd107d06..c09e74d6533a4 100644 --- a/doc/Project.toml +++ b/doc/Project.toml @@ -1,2 +1,3 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +DocumenterLaTeX = "cd674d7a-5f81-5cf3-af33-235ef1834b99" diff --git a/doc/make.jl b/doc/make.jl index 7a7e8cf1953dd..404003ddc4d23 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -6,7 +6,7 @@ pushfirst!(DEPOT_PATH, joinpath(@__DIR__, "deps")) using Pkg Pkg.instantiate() -using Documenter +using Documenter, DocumenterLaTeX # Include the `build_sysimg` file. @@ -153,6 +153,17 @@ let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures) end +const format = if render_pdf + LaTeX( + platform = "texplatform=docker" in ARGS ? "docker" : "native" + ) +else + Documenter.HTML( + prettyurls = ("deploy" in ARGS), + canonical = ("deploy" in ARGS) ? "https://docs.julialang.org/en/v1/" : nothing, + ) +end + makedocs( build = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en"), modules = [Base, Core, BuildSysImg, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], @@ -162,13 +173,11 @@ makedocs( linkcheck_ignore = ["https://bugs.kde.org/show_bug.cgi?id=136779"], # fails to load from nanosoldier? strict = true, checkdocs = :none, - format = render_pdf ? :latex : :html, + format = format, sitename = "The Julia Language", authors = "The Julia Project", analytics = "UA-28835595-6", pages = PAGES, - html_prettyurls = ("deploy" in ARGS), - html_canonical = ("deploy" in ARGS) ? "https://docs.julialang.org/en/v1/" : nothing, assets = ["assets/julia-manual.css", ] ) From f4cd4030080fb64b824993fd2b34171464ceadc3 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Tue, 11 Dec 2018 23:39:17 +0100 Subject: [PATCH 07/41] fix bug with max_values in union! (#30315) (cherry picked from commit f49cb42fb06492765f6320f3161e8363be5b7ada) --- base/abstractset.jl | 6 ++++-- test/sets.jl | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/base/abstractset.jl b/base/abstractset.jl index 100383a126a1d..03038ff6364da 100644 --- a/base/abstractset.jl +++ b/base/abstractset.jl @@ -67,9 +67,11 @@ function union!(s::AbstractSet, sets...) end max_values(::Type) = typemax(Int) -max_values(T::Type{<:Union{Nothing,BitIntegerSmall}}) = 1 << (8*sizeof(T)) -max_values(T::Union) = max(max_values(T.a), max_values(T.b)) +max_values(T::Union{map(X -> Type{X}, BitIntegerSmall_types)...}) = 1 << (8*sizeof(T)) +# saturated addition to prevent overflow with typemax(Int) +max_values(T::Union) = max(max_values(T.a), max_values(T.b), max_values(T.a) + max_values(T.b)) max_values(::Type{Bool}) = 2 +max_values(::Type{Nothing}) = 1 function union!(s::AbstractSet{T}, itr) where T haslength(itr) && sizehint!(s, length(s) + length(itr)) diff --git a/test/sets.jl b/test/sets.jl index daddf0bc7bb8b..594c1f0f89087 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -613,6 +613,27 @@ end end end +@testset "optimized union! with max_values" begin + # issue #30315 + T = Union{Nothing, Bool} + @test Base.max_values(T) == 3 + d = Set{T}() + union!(d, (nothing, true, false)) + @test length(d) == 3 + @test d == Set((nothing, true, false)) + @test nothing in d + @test true in d + @test false in d + + for X = (Int8, Int16, Int32, Int64) + @test Base.max_values(Union{Nothing, X}) == (sizeof(X) < sizeof(Int) ? + 2^(8*sizeof(X)) + 1 : + typemax(Int)) + end + # this does not account for non-empty intersections of the unioned types + @test Base.max_values(Union{Int8,Int16}) == 2^8 + 2^16 +end + struct OpenInterval{T} lower::T upper::T From e2aebba4c4bb047b2d1eb781d33a0fc357623f52 Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Wed, 12 Dec 2018 10:50:03 +0100 Subject: [PATCH 08/41] Force specialization on the type argument of `_similar_for` (#30331) (cherry picked from commit 891e2abdd778ae053451863f0ee81950834f881c) --- base/array.jl | 10 ++++++---- base/dict.jl | 3 ++- base/set.jl | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/base/array.jl b/base/array.jl index a10678d44b1c6..dcea65bcadba6 100644 --- a/base/array.jl +++ b/base/array.jl @@ -528,10 +528,12 @@ function _collect(::Type{T}, itr, isz::SizeUnknown) where T end # make a collection similar to `c` and appropriate for collecting `itr` -_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0) -_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer)) -_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, axes(itr)) -_similar_for(c, T, itr, isz) = similar(c, T) +_similar_for(c::AbstractArray, ::Type{T}, itr, ::SizeUnknown) where {T} = similar(c, T, 0) +_similar_for(c::AbstractArray, ::Type{T}, itr, ::HasLength) where {T} = + similar(c, T, Int(length(itr)::Integer)) +_similar_for(c::AbstractArray, ::Type{T}, itr, ::HasShape) where {T} = + similar(c, T, axes(itr)) +_similar_for(c, ::Type{T}, itr, isz) where {T} = similar(c, T) """ collect(collection) diff --git a/base/dict.jl b/base/dict.jl index 1c1d0f51b9cd7..cf25738d95cd5 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -760,4 +760,5 @@ isempty(t::ImmutableDict) = !isdefined(t, :parent) empty(::ImmutableDict, ::Type{K}, ::Type{V}) where {K, V} = ImmutableDict{K,V}() _similar_for(c::Dict, ::Type{Pair{K,V}}, itr, isz) where {K, V} = empty(c, K, V) -_similar_for(c::AbstractDict, T, itr, isz) = throw(ArgumentError("for AbstractDicts, similar requires an element type of Pair;\n if calling map, consider a comprehension instead")) +_similar_for(c::AbstractDict, ::Type{T}, itr, isz) where {T} = + throw(ArgumentError("for AbstractDicts, similar requires an element type of Pair;\n if calling map, consider a comprehension instead")) diff --git a/base/set.jl b/base/set.jl index ab64837aa9ce1..583d5e350536e 100644 --- a/base/set.jl +++ b/base/set.jl @@ -34,7 +34,7 @@ empty(s::AbstractSet{T}, ::Type{U}=T) where {T,U} = Set{U}() # by default, a Set is returned emptymutable(s::AbstractSet{T}, ::Type{U}=T) where {T,U} = Set{U}() -_similar_for(c::AbstractSet, T, itr, isz) = empty(c, T) +_similar_for(c::AbstractSet, ::Type{T}, itr, isz) where {T} = empty(c, T) function show(io::IO, s::Set) print(io, "Set(") From 43d99bb846b812a13c55ba63f72a5a75f44ea00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Wed, 12 Dec 2018 20:06:25 +0100 Subject: [PATCH 09/41] added doc for AbstractChannel (#30347) (cherry picked from commit dda64505b083704f190040333135807d96fe1743) --- base/channels.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base/channels.jl b/base/channels.jl index 784a7d6de9781..8895ca50e1bf1 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -1,5 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +""" + AbstractChannel{T} + +Representation of a channel passing objects of type `T`. +""" abstract type AbstractChannel{T} end """ From 43d1d9c94e1f00a02e8cbeb031ad5f65c3149d30 Mon Sep 17 00:00:00 2001 From: Don March Date: Wed, 12 Dec 2018 18:41:56 -0500 Subject: [PATCH 10/41] Copy editing in "Environment variables" docs (#30330) (cherry picked from commit 99b7b75ed08c71aa4be0a64111909cc5d2032e33) --- doc/src/manual/environment-variables.md | 40 ++++++++++++------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index d92260dd376ff..7bc47ba43ed2a 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -1,25 +1,26 @@ # Environment Variables -Julia may be configured with a number of environment variables, either in the -usual way of the operating system, or in a portable way from within Julia. -Suppose you want to set the environment variable `JULIA_EDITOR` to -`vim`, then either type `ENV["JULIA_EDITOR"] = "vim"` for instance in the REPL -to make this change on a case by case basis, or add the same to the user -configuration file `~/.julia/config/startup.jl` in the user's home directory to have -a permanent effect. The current value of the same environment variable is +Julia can be configured with a number of environment variables, set either in +the usual way for each operating system, or in a portable way from within Julia. +Supposing that you want to set the environment variable `JULIA_EDITOR` to `vim`, +you can type `ENV["JULIA_EDITOR"] = "vim"` (for instance, in the REPL) to make +this change on a case by case basis, or add the same to the user configuration +file `~/.julia/config/startup.jl` in the user's home directory to have a +permanent effect. The current value of the same environment variable can be determined by evaluating `ENV["JULIA_EDITOR"]`. The environment variables that Julia uses generally start with `JULIA`. If -[`InteractiveUtils.versioninfo`](@ref) is called with `verbose` equal to `true`, then the +[`InteractiveUtils.versioninfo`](@ref) is called with the keyword `verbose=true`, then the output will list defined environment variables relevant for Julia, including those for which `JULIA` appears in the name. !!! note - Some variables, such as `JULIA_NUM_THREADS` and `JULIA_PROJECT` need to be set before Julia + Some variables, such as `JULIA_NUM_THREADS` and `JULIA_PROJECT`, need to be set before Julia starts, therefore adding these to `~/.julia/config/startup.jl` is too late in the startup process. - These must either be set manually before launching Julia through bash with - `export JULIA_NUM_THREADS=4` etc. or added to `-/.bashrc` and/or `~/.bash_profile` to achieve persistence. + In Bash, environment variables can either be set manually by running, e.g., + `export JULIA_NUM_THREADS=4` before starting Julia, or by adding the same command to + `-/.bashrc` or `~/.bash_profile` to set the variable each time Bash is started. ## File locations @@ -76,7 +77,7 @@ and a global configuration search path of A directory path that points to the current Julia project. Setting this environment variable has the same effect as specifying the `--project` start-up -option, but `--project` has higher precedence. If the variable is set to `@.`, +option, but `--project` has higher precedence. If the variable is set to `@.` then Julia tries to find a project directory that contains `Project.toml` or `JuliaProject.toml` file from the current directory and its parents. See also the chapter on [Code Loading](@ref). @@ -88,8 +89,8 @@ the chapter on [Code Loading](@ref). ### `JULIA_LOAD_PATH` A separated list of absolute paths that are to be appended to the variable -[`LOAD_PATH`](@ref). (In Unix-like systems, the path separator is `:`; in -Windows systems, the path separator is `;`.) The `LOAD_PATH` variable is where +[`LOAD_PATH`](@ref). (In Unix-like systems, `:` is the path separator; in +Windows systems, `;` is the path separator.) The `LOAD_PATH` variable is where [`Base.require`](@ref) and `Base.load_in_path()` look for code; it defaults to the absolute path `$JULIA_HOME/../share/julia/stdlib/v$(VERSION.major).$(VERSION.minor)` so that, @@ -185,7 +186,7 @@ affinitized. Otherwise, Julia lets the operating system handle thread policy. Environment variables that determine how REPL output should be formatted at the terminal. Generally, these variables should be set to [ANSI terminal escape sequences](http://ascii-table.com/ansi-escape-sequences.php). Julia provides -a high-level interface with much of the same functionality: see the section on +a high-level interface with much of the same functionality; see the section on [The Julia REPL](@ref). ### `JULIA_ERROR_COLOR` @@ -283,11 +284,10 @@ event listener for just-in-time (JIT) profiling. This environment variable only has an effect if Julia was compiled with JIT profiling support, using either - -* Intel's [VTune™ Amplifier](https://software.intel.com/en-us/intel-vtune-amplifier-xe) - (`USE_INTEL_JITEVENTS` set to `1` in the build configuration), or -* [OProfile](http://oprofile.sourceforge.net/news/) (`USE_OPROFILE_JITEVENTS` set to `1` - in the build configuration). + * Intel's [VTune™ Amplifier](https://software.intel.com/en-us/intel-vtune-amplifier-xe) + (`USE_INTEL_JITEVENTS` set to `1` in the build configuration), or + * [OProfile](http://oprofile.sourceforge.net/news/) (`USE_OPROFILE_JITEVENTS` set to `1` + in the build configuration). ### `JULIA_LLVM_ARGS` From 96f82b508a1dec7e9b86464268f8d91c4476e290 Mon Sep 17 00:00:00 2001 From: Daniel Karrasch Date: Thu, 13 Dec 2018 11:23:44 +0100 Subject: [PATCH 11/41] [SparseArrays] Respect order of mul in (l)mul!(::Diagonal,::Sparse) (#30163) * order of mul in (l)mul!(::Diagonal,::Sparse) add tests for non-commutative mul * Create Quaternions.jl remove quaternions from generic tests (cherry picked from commit 3ee8798fc428bdbb9583a2c007e63be0c298d04a) --- stdlib/LinearAlgebra/test/generic.jl | 31 ++++--------------------- stdlib/SparseArrays/src/linalg.jl | 6 ++--- stdlib/SparseArrays/test/sparse.jl | 24 ++++++++++++++++++++ test/testhelpers/Quaternions.jl | 34 ++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 30 deletions(-) create mode 100644 test/testhelpers/Quaternions.jl diff --git a/stdlib/LinearAlgebra/test/generic.jl b/stdlib/LinearAlgebra/test/generic.jl index 13d18e1d29b23..5e92b0d24fd24 100644 --- a/stdlib/LinearAlgebra/test/generic.jl +++ b/stdlib/LinearAlgebra/test/generic.jl @@ -3,33 +3,10 @@ module TestGeneric using Test, LinearAlgebra, Random -import Base: -, *, /, \ - -# A custom Quaternion type with minimal defined interface and methods. -# Used to test mul and mul! methods to show non-commutativity. -struct Quaternion{T<:Real} <: Number - s::T - v1::T - v2::T - v3::T -end -Quaternion(s::Real, v1::Real, v2::Real, v3::Real) = Quaternion(promote(s, v1, v2, v3)...) -Base.abs2(q::Quaternion) = q.s*q.s + q.v1*q.v1 + q.v2*q.v2 + q.v3*q.v3 -Base.abs(q::Quaternion) = sqrt(abs2(q)) -Base.real(::Type{Quaternion{T}}) where {T} = T -Base.conj(q::Quaternion) = Quaternion(q.s, -q.v1, -q.v2, -q.v3) -Base.isfinite(q::Quaternion) = isfinite(q.s) & isfinite(q.v1) & isfinite(q.v2) & isfinite(q.v3) - -(-)(ql::Quaternion, qr::Quaternion) = - Quaternion(ql.s - qr.s, ql.v1 - qr.v1, ql.v2 - qr.v2, ql.v3 - qr.v3) -(*)(q::Quaternion, w::Quaternion) = Quaternion(q.s*w.s - q.v1*w.v1 - q.v2*w.v2 - q.v3*w.v3, - q.s*w.v1 + q.v1*w.s + q.v2*w.v3 - q.v3*w.v2, - q.s*w.v2 - q.v1*w.v3 + q.v2*w.s + q.v3*w.v1, - q.s*w.v3 + q.v1*w.v2 - q.v2*w.v1 + q.v3*w.s) -(*)(q::Quaternion, r::Real) = Quaternion(q.s*r, q.v1*r, q.v2*r, q.v3*r) -(*)(q::Quaternion, b::Bool) = b * q # remove method ambiguity -(/)(q::Quaternion, w::Quaternion) = q * conj(w) * (1.0 / abs2(w)) -(\)(q::Quaternion, w::Quaternion) = conj(q) * w * (1.0 / abs2(q)) + +const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") +isdefined(Main, :Quaternions) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "Quaternions.jl")) +using .Main.Quaternions Random.seed!(123) diff --git a/stdlib/SparseArrays/src/linalg.jl b/stdlib/SparseArrays/src/linalg.jl index f368eb277b162..d6a97e759b8d9 100644 --- a/stdlib/SparseArrays/src/linalg.jl +++ b/stdlib/SparseArrays/src/linalg.jl @@ -992,7 +992,7 @@ function mul!(C::SparseMatrixCSC, D::Diagonal{T, <:Vector}, A::SparseMatrixCSC) Arowval = A.rowval resize!(Cnzval, length(Anzval)) for col = 1:n, p = A.colptr[col]:(A.colptr[col+1]-1) - @inbounds Cnzval[p] = Anzval[p] * b[Arowval[p]] + @inbounds Cnzval[p] = b[Arowval[p]] * Anzval[p] end C end @@ -1028,7 +1028,7 @@ function rmul!(A::SparseMatrixCSC, D::Diagonal) (n == size(D, 1)) || throw(DimensionMismatch()) Anzval = A.nzval @inbounds for col = 1:n, p = A.colptr[col]:(A.colptr[col + 1] - 1) - Anzval[p] *= D.diag[col] + Anzval[p] = Anzval[p] * D.diag[col] end return A end @@ -1039,7 +1039,7 @@ function lmul!(D::Diagonal, A::SparseMatrixCSC) Anzval = A.nzval Arowval = A.rowval @inbounds for col = 1:n, p = A.colptr[col]:(A.colptr[col + 1] - 1) - Anzval[p] *= D.diag[Arowval[p]] + Anzval[p] = D.diag[Arowval[p]] * Anzval[p] end return A end diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 416cbffac27cf..c57695062c48c 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -364,6 +364,10 @@ end @test_throws DimensionMismatch dot(sprand(5,5,0.2),sprand(5,6,0.2)) end +const BASE_TEST_PATH = joinpath(Sys.BINDIR, "..", "share", "julia", "test") +isdefined(Main, :Quaternions) || @eval Main include(joinpath($(BASE_TEST_PATH), "testhelpers", "Quaternions.jl")) +using .Main.Quaternions + sA = sprandn(3, 7, 0.5) sC = similar(sA) dA = Array(sA) @@ -396,6 +400,26 @@ dA = Array(sA) @test_throws DimensionMismatch rdiv!(copy(sAt), Diagonal(fill(1., length(b)+1))) @test_throws LinearAlgebra.SingularException rdiv!(copy(sAt), Diagonal(zeros(length(b)))) end + + @testset "non-commutative multiplication" begin + # non-commutative multiplication + Avals = Quaternion.(randn(10), randn(10), randn(10), randn(10)) + sA = sparse(rand(1:3, 10), rand(1:7, 10), Avals, 3, 7) + sC = copy(sA) + dA = Array(sA) + + b = Quaternion.(randn(7), randn(7), randn(7), randn(7)) + D = Diagonal(b) + @test Array(sA * D) ≈ dA * D + @test rmul!(copy(sA), D) ≈ dA * D + @test mul!(sC, copy(sA), D) ≈ dA * D + + b = Quaternion.(randn(3), randn(3), randn(3), randn(3)) + D = Diagonal(b) + @test Array(D * sA) ≈ D * dA + @test lmul!(D, copy(sA)) ≈ D * dA + @test mul!(sC, D, copy(sA)) ≈ D * dA + end end @testset "copyto!" begin diff --git a/test/testhelpers/Quaternions.jl b/test/testhelpers/Quaternions.jl new file mode 100644 index 0000000000000..0920b9dea00c6 --- /dev/null +++ b/test/testhelpers/Quaternions.jl @@ -0,0 +1,34 @@ +module Quaternions + +export Quaternion + +# A custom Quaternion type with minimal defined interface and methods. +# Used to test mul and mul! methods to show non-commutativity. +struct Quaternion{T<:Real} <: Number + s::T + v1::T + v2::T + v3::T +end +Quaternion(s::Real, v1::Real, v2::Real, v3::Real) = Quaternion(promote(s, v1, v2, v3)...) +Base.abs2(q::Quaternion) = q.s*q.s + q.v1*q.v1 + q.v2*q.v2 + q.v3*q.v3 +Base.abs(q::Quaternion) = sqrt(abs2(q)) +Base.real(::Type{Quaternion{T}}) where {T} = T +Base.conj(q::Quaternion) = Quaternion(q.s, -q.v1, -q.v2, -q.v3) +Base.isfinite(q::Quaternion) = isfinite(q.s) & isfinite(q.v1) & isfinite(q.v2) & isfinite(q.v3) +Base.zero(::Type{Quaternion{T}}) where T = Quaternion{T}(zero(T), zero(T), zero(T), zero(T)) + +Base.:(+)(ql::Quaternion, qr::Quaternion) = + Quaternion(ql.s + qr.s, ql.v1 + qr.v1, ql.v2 + qr.v2, ql.v3 + qr.v3) +Base.:(-)(ql::Quaternion, qr::Quaternion) = + Quaternion(ql.s - qr.s, ql.v1 - qr.v1, ql.v2 - qr.v2, ql.v3 - qr.v3) +Base.:(*)(q::Quaternion, w::Quaternion) = Quaternion(q.s*w.s - q.v1*w.v1 - q.v2*w.v2 - q.v3*w.v3, + q.s*w.v1 + q.v1*w.s + q.v2*w.v3 - q.v3*w.v2, + q.s*w.v2 - q.v1*w.v3 + q.v2*w.s + q.v3*w.v1, + q.s*w.v3 + q.v1*w.v2 - q.v2*w.v1 + q.v3*w.s) +Base.:(*)(q::Quaternion, r::Real) = Quaternion(q.s*r, q.v1*r, q.v2*r, q.v3*r) +Base.:(*)(q::Quaternion, b::Bool) = b * q # remove method ambiguity +Base.:(/)(q::Quaternion, w::Quaternion) = q * conj(w) * (1.0 / abs2(w)) +Base.:(\)(q::Quaternion, w::Quaternion) = conj(q) * w * (1.0 / abs2(q)) + +end From 33f48cdb1637762257dcbef3cf480daf5b04a6d1 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 14 Dec 2018 10:41:51 +0100 Subject: [PATCH 12/41] fix reinterpret for 0-dimensional arrays (#30376) (cherry picked from commit c3799003d769c434a2507ec472aa3f80f6c39317) --- base/reinterpretarray.jl | 4 +++- test/reinterpretarray.jl | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 2ea21ec981b56..8d7a175d8252d 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -33,8 +33,8 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S, N}} <: AbstractArray{T, N} isbitstype(T) || throwbits(S, T, T) isbitstype(S) || throwbits(S, T, S) (N != 0 || sizeof(T) == sizeof(S)) || throwsize0(S, T) - ax1 = axes(a)[1] if N != 0 && sizeof(S) != sizeof(T) + ax1 = axes(a)[1] dim = length(ax1) rem(dim*sizeof(S),sizeof(T)) == 0 || thrownonint(S, T, dim) first(ax1) == 1 || throwaxes1(S, T, ax1) @@ -74,6 +74,7 @@ function size(a::ReinterpretArray{T,N,S} where {N}) where {T,S} size1 = div(psize[1]*sizeof(S), sizeof(T)) tuple(size1, tail(psize)...) end +size(a::ReinterpretArray{T,0}) where {T} = () function axes(a::ReinterpretArray{T,N,S} where {N}) where {T,S} paxs = axes(a.parent) @@ -81,6 +82,7 @@ function axes(a::ReinterpretArray{T,N,S} where {N}) where {T,S} size1 = div(l*sizeof(S), sizeof(T)) tuple(oftype(paxs[1], f:f+size1-1), tail(paxs)...) end +axes(a::ReinterpretArray{T,0}) where {T} = () elsize(::Type{<:ReinterpretArray{T}}) where {T} = sizeof(T) unsafe_convert(::Type{Ptr{T}}, a::ReinterpretArray{T,N,S} where N) where {T,S} = Ptr{T}(unsafe_convert(Ptr{S},a.parent)) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index bdb588433dba9..589106f87ab1a 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -160,3 +160,12 @@ let a = [0.1 0.2; 0.3 0.4], at = reshape([(i,i+1) for i = 1:2:8], 2, 2) r = reinterpret(Int, vt) @test r == OffsetArray(reshape(1:8, 2, 2, 2), (0, offsetvt...)) end + +# Test 0-dimensional Arrays +A = zeros(UInt32) +B = reinterpret(Int32,A) +@test size(B) == () +@test axes(B) == () +B[] = Int32(5) +@test B[] === Int32(5) +@test A[] === UInt32(5) From 0ab6919c327de2a8de8cdef24d465f7fa0db99d8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 16 Dec 2018 21:09:56 -0500 Subject: [PATCH 13/41] stacktrace: prevent OOB-error in sysimage lookup (#30369) Previously, with a multi-versioned system image, there might be additional entries at the end of the clone list that do not correspond to an actual method (such as jlplt thunks). Also some code cleanup for clarity. fix #28648 (cherry picked from commit e51a7075d74e86274d694b9b9f5e475b57c05439) --- src/debuginfo.cpp | 3 ++- src/staticdata.c | 1 + stdlib/Profile/src/Profile.jl | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 15c33f86524cd..06664623dc7f5 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -1101,7 +1101,8 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip for (size_t i = 0; i < sysimg_fptrs.nclones; i++) { if (diff == sysimg_fptrs.clone_offsets[i]) { uint32_t idx = sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask; - frame0->linfo = sysimg_fvars_linfo[idx]; + if (idx < sysimg_fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) + frame0->linfo = sysimg_fvars_linfo[idx]; break; } } diff --git a/src/staticdata.c b/src/staticdata.c index 0f1aa9d1356ae..85496b6d07ed3 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1015,6 +1015,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s) for (i = 0; i < sysimg_fvars_max; i++) { uintptr_t val = (uintptr_t)&linfos[i]; uint32_t offset = load_uint32(&val); + linfos[i] = NULL; if (offset != 0) { int specfunc = 1; if (offset & ((uintptr_t)1 << (8 * sizeof(uint32_t) - 1))) { diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 5cb34e132942f..8ae9542c2c2f4 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -514,6 +514,7 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI # jump forward to the end of the inlining chain # avoiding an extra (slow) lookup of `ip` in `lidict` # and an extra chain of them in `down` + # note that we may even have this === parent (if we're ignoring this frame ip) this = builder_value[fastkey] let this = this while this !== parent @@ -532,8 +533,7 @@ function tree!(root::StackFrameTree{T}, all::Vector{UInt64}, lidict::Union{LineI frame = (frames isa Vector ? frames[i] : frames) !C && frame.from_c && continue key = (T === UInt64 ? ip : frame) - down = parent.down - this = get!(down, key) do + this = get!(parent.down, key) do return StackFrameTree{T}() end this.frame = frame From 3c9592d1917d19db0dc79c6339b7cb4418de4cac Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 19 Dec 2018 20:42:28 -0500 Subject: [PATCH 14/41] generalize sparse matrix slicing to integer types (#30319) (cherry picked from commit 072ad7db4a0bdfa141713ae59dbf6b0706302481) --- stdlib/SparseArrays/src/sparsematrix.jl | 4 ++-- stdlib/SparseArrays/test/sparse.jl | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/stdlib/SparseArrays/src/sparsematrix.jl b/stdlib/SparseArrays/src/sparsematrix.jl index 02c975168c6ec..ff288017571d1 100644 --- a/stdlib/SparseArrays/src/sparsematrix.jl +++ b/stdlib/SparseArrays/src/sparsematrix.jl @@ -1968,8 +1968,8 @@ function getindex_cols(A::SparseMatrixCSC{Tv,Ti}, J::AbstractVector) where {Tv,T return SparseMatrixCSC(m, nJ, colptrS, rowvalS, nzvalS) end -getindex_traverse_col(::AbstractUnitRange, lo::Int, hi::Int) = lo:hi -getindex_traverse_col(I::StepRange, lo::Int, hi::Int) = step(I) > 0 ? (lo:1:hi) : (hi:-1:lo) +getindex_traverse_col(::AbstractUnitRange, lo::Integer, hi::Integer) = lo:hi +getindex_traverse_col(I::StepRange, lo::Integer, hi::Integer) = step(I) > 0 ? (lo:1:hi) : (hi:-1:lo) function getindex(A::SparseMatrixCSC{Tv,Ti}, I::AbstractRange, J::AbstractVector) where {Tv,Ti<:Integer} @assert !has_offset_axes(A, I, J) diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index c57695062c48c..5e6d82bca5d5b 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -733,6 +733,8 @@ end @test ss116[:,:] == copy(ss116) + @test convert(SparseMatrixCSC{Float32,Int32}, sd116)[2:5,:] == convert(SparseMatrixCSC{Float32,Int32}, sd116[2:5,:]) + # range indexing @test Array(ss116[i,:]) == aa116[i,:] @test Array(ss116[:,j]) == aa116[:,j] From 834503f69cba4363e91cfc675e6052869fd1b01b Mon Sep 17 00:00:00 2001 From: Markus Kuhn Date: Thu, 20 Dec 2018 07:34:18 +0000 Subject: [PATCH 15/41] Base.worker_timeout() mention in manual (#30439) The manual mentions at https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_WORKER_TIMEOUT-1 a function Base.worker_timeout() but the implementation has instead only a function Distributed.worker_timeout() (cherry picked from commit 258e08a605a35a34dcc181b74f2b311459ee84c4) --- doc/src/manual/environment-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index 7bc47ba43ed2a..c410adce6e5dd 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -151,7 +151,7 @@ logical CPU cores available. ### `JULIA_WORKER_TIMEOUT` -A [`Float64`](@ref) that sets the value of `Base.worker_timeout()` (default: `60.0`). +A [`Float64`](@ref) that sets the value of `Distributed.worker_timeout()` (default: `60.0`). This function gives the number of seconds a worker process will wait for a master process to establish a connection before dying. From ded25d37be2c18f98b81be2ab726923359d5970d Mon Sep 17 00:00:00 2001 From: Raghvendra Gupta Date: Wed, 26 Dec 2018 22:56:06 +0530 Subject: [PATCH 16/41] Fix DimensionMismatch in SparseMatrixCSC assignment (#30507) Use _setindex! to fix DimensionMismatch in sparse assignment Fixes #28963 (cherry picked from commit b73a74606c845afe7d15d0da9f030efd72bcb720) --- stdlib/SparseArrays/src/sparsematrix.jl | 1 + stdlib/SparseArrays/test/sparse.jl | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/stdlib/SparseArrays/src/sparsematrix.jl b/stdlib/SparseArrays/src/sparsematrix.jl index ff288017571d1..165d2d6a53685 100644 --- a/stdlib/SparseArrays/src/sparsematrix.jl +++ b/stdlib/SparseArrays/src/sparsematrix.jl @@ -2564,6 +2564,7 @@ function setindex!(A::SparseMatrixCSC{Tv,Ti}, V::AbstractVecOrMat, Ix::Union{Int @assert !has_offset_axes(A, V, Ix, Jx) (I, J) = Base.ensure_indexable(to_indices(A, (Ix, Jx))) checkbounds(A, I, J) + Base._setindex!(IndexStyle(A), A, V, to_indices(A, (Ix, Jx))...) B = _to_same_csc(A, V, I, J) issortedI = issorted(I) diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 5e6d82bca5d5b..680e94c57b7ec 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -181,6 +181,10 @@ end end end +@testset "Issue #28963" begin + @test_throws DimensionMismatch (spzeros(10,10)[:, :] = sprand(10,20,0.5)) +end + @testset "matrix-vector multiplication (non-square)" begin for i = 1:5 a = sprand(10, 5, 0.5) From 0195be052975b85a988692e5801fbf052b5db58c Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Fri, 28 Dec 2018 22:06:04 -0500 Subject: [PATCH 17/41] faster mapfoldl for tuples (#30471) (cherry picked from commit 6dc205a521d789c7d3b34e3508ad2b7f75284ed0) --- base/tuple.jl | 12 ++++++++++++ test/tuple.jl | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/base/tuple.jl b/base/tuple.jl index a1889b79b6faa..358db6946fb7e 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -209,6 +209,18 @@ function map(f, t1::Any16, t2::Any16, ts::Any16...) (A...,) end +# mapafoldl, based on afold in operators.jl +mapafoldl(F,op,a) = a +mapafoldl(F,op,a,b) = op(a,F(b)) +mapafoldl(F,op,a,b,c...) = mapafoldl(F, op, op(a,F(b)), c...) +function mapafoldl(F,op,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,qs...) + y = op(op(op(op(op(op(op(op(op(op(op(op(op(op(op(a,F(b)),F(c)),F(d)),F(e)),F(f)),F(g)),F(h)),F(i)),F(j)),F(k)),F(l)),F(m)),F(n)),F(o)),F(p)) + for x in qs; y = op(y,F(x)); end + y +end +mapfoldl_impl(f, op, nt::NamedTuple{(:init,)}, t::Tuple) = mapafoldl(f, op, nt.init, t...) +mapfoldl_impl(f, op, nt::NamedTuple{()}, t::Tuple) = mapafoldl(f, op, f(t[1]), tail(t)...) +mapfoldl_impl(f, op, nt::NamedTuple{()}, t::Tuple{}) = mapreduce_empty_iter(f, op, t, IteratorEltype(t)) # type-stable padding fill_to_length(t::NTuple{N,Any}, val, ::Val{N}) where {N} = t diff --git a/test/tuple.jl b/test/tuple.jl index 3a22f3b089557..9b56ccff20fbe 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -237,6 +237,15 @@ end end end +@testset "mapfoldl" begin + @test (((1=>2)=>3)=>4) == foldl(=>, (1,2,3,4)) == + mapfoldl(identity, =>, (1,2,3,4)) == mapfoldl(abs, =>, (-1,-2,-3,-4)) + @test mapfoldl(abs, =>, (-1,-2,-3,-4), init=-10) == ((((-10=>1)=>2)=>3)=>4) + @test mapfoldl(abs, =>, (), init=-10) == -10 + @test mapfoldl(abs, Pair{Any,Any}, (-30:-1...,)) == mapfoldl(abs, Pair{Any,Any}, [-30:-1...,]) + @test_throws ArgumentError mapfoldl(abs, =>, ()) +end + @testset "comparison and hash" begin @test isequal((), ()) @test isequal((1,2,3), (1,2,3)) From 2d1c9259f76324429b527ef2a2f0d8092bf0e000 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 17 Dec 2018 11:39:38 -0500 Subject: [PATCH 18/41] fix #30394, an unsoundness in ml_matches (#30396) This fixes a corner case where a bug is caused, counter-intuitively, by an over-estimated intersection. We have method signatures A and B, with Amax_valid = ml->max_world; } } + // In some corner cases type intersection is conservative and returns something + // for intersect(A, B) even though A is a dispatch tuple and !(A <: B). + // For dispatch purposes in such a case we know there's no match. This check + // fixes issue #30394. + if (jl_is_dispatch_tupletype(closure->match.type) && !closure->match.issubty) + return 1; // a method is shadowed if type <: S <: m->sig where S is the // signature of another applicable method /* diff --git a/test/compiler/compiler.jl b/test/compiler/compiler.jl index 01b7000a6a68a..8ff37af09aac9 100644 --- a/test/compiler/compiler.jl +++ b/test/compiler/compiler.jl @@ -2068,3 +2068,24 @@ function f29036(s, i) val end @test Base.return_types(f29036, (String, Int)) == Any[Char] + +# issue #30394 +mutable struct Base30394 + a::Int +end + +mutable struct Foo30394 + foo_inner::Base30394 + Foo30394() = new(Base30394(1)) +end + +mutable struct Foo30394_2 + foo_inner::Foo30394 + Foo30394_2() = new(Foo30394()) +end + +f30394(foo::T1, ::Type{T2}) where {T2, T1 <: T2} = foo + +f30394(foo, T2) = f30394(foo.foo_inner, T2) + +@test Base.return_types(f30394, (Foo30394_2, Type{Base30394})) == Any[Base30394] From aede02488b04d24d70e69c1616e7edfe43f2ad6f Mon Sep 17 00:00:00 2001 From: Ben Arthur Date: Sat, 29 Dec 2018 07:48:42 -0600 Subject: [PATCH 19/41] cluster manager fixes (#30172) * kill workers which don't launch properly * don't emit spurious error messages * document how to asynchronously launch workers (cherry picked from commit 121e814bfeec8ac0a45da2c4b5c87189366e1aec) --- stdlib/Distributed/src/cluster.jl | 51 ++++++++++++++++++--- stdlib/Distributed/test/distributed_exec.jl | 2 +- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/stdlib/Distributed/src/cluster.jl b/stdlib/Distributed/src/cluster.jl index e1d4bae511d91..288e55197be0a 100644 --- a/stdlib/Distributed/src/cluster.jl +++ b/stdlib/Distributed/src/cluster.jl @@ -245,6 +245,11 @@ function redirect_worker_output(ident, stream) end end +struct LaunchWorkerError <: Exception + msg::String +end + +Base.showerror(io::IO, e::LaunchWorkerError) = print(io, e.msg) # The default TCP transport relies on the worker listening on a free # port available and printing its bind address and port. @@ -276,7 +281,7 @@ function read_worker_host_port(io::IO) conninfo = fetch(readtask) if isempty(conninfo) && !isopen(io) - error("Unable to read host:port string from worker. Launch command exited with error?") + throw(LaunchWorkerError("Unable to read host:port string from worker. Launch command exited with error?")) end ntries -= 1 @@ -290,13 +295,13 @@ function read_worker_host_port(io::IO) end close(io) if ntries > 0 - error("Timed out waiting to read host:port string from worker.") + throw(LaunchWorkerError("Timed out waiting to read host:port string from worker.")) else - error("Unexpected output from worker launch command. Host:port string not found.") + throw(LaunchWorkerError("Unexpected output from worker launch command. Host:port string not found.")) end finally for line in leader - println("\tFrom failed worker startup:\t", line) + println("\tFrom worker startup:\t", line) end end end @@ -358,6 +363,34 @@ the package `ClusterManagers.jl`. The number of seconds a newly launched worker waits for connection establishment from the master can be specified via variable `JULIA_WORKER_TIMEOUT` in the worker process's environment. Relevant only when using TCP/IP as transport. + +To launch workers without blocking the REPL, or the containing function +if launching workers programmatically, execute `addprocs` in its own task. + +# Examples + +``` +# On busy clusters, call `addprocs` asynchronously +t = @async addprocs(...) +``` + +``` +# Utilize workers as and when they come online +if nprocs() > 1 # Ensure at least one new worker is available + .... # perform distributed execution +end +``` + +``` +# Retrieve newly launched worker IDs, or any error messages +if istaskdone(t) # Check if `addprocs` has completed to ensure `fetch` doesn't block + if nworkers() == N + new_pids = fetch(t) + else + fetch(t) + end + end +``` """ function addprocs(manager::ClusterManager; kwargs...) init_multi() @@ -503,9 +536,13 @@ function create_worker(manager, wconfig) local r_s, w_s try (r_s, w_s) = connect(manager, w.id, wconfig) - catch e - deregister_worker(w.id) - rethrow(e) + catch ex + try + deregister_worker(w.id) + kill(manager, w.id, wconfig) + finally + rethrow(ex) + end end w = Worker(w.id, r_s, w_s, manager; config=wconfig) diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index 383a6783a311d..1248fa840fc87 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1129,7 +1129,7 @@ for (addp_testf, expected_errstr, env) in testruns old_stdout = stdout stdout_out, stdout_in = redirect_stdout() stdout_txt = @async filter!(readlines(stdout_out)) do s - return !startswith(s, "\tFrom failed worker startup:\t") + return !startswith(s, "\tFrom worker startup:\t") end try withenv(env...) do From e4a7db2680171f113381742c8252e0b7cb83d662 Mon Sep 17 00:00:00 2001 From: Raghvendra Gupta Date: Mon, 17 Dec 2018 20:20:13 +0530 Subject: [PATCH 20/41] Fix #30006, getindex accessing fields that might not exist (#30405) * Fix #30006, range getindex accessing fields that might not exist * Add tests for #30006 (cherry picked from commit 64133f68a68a2bb52a8908bab25c32150a7e84fd) --- base/range.jl | 4 ++-- stdlib/SparseArrays/test/sparse.jl | 4 ++++ test/ranges.jl | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/base/range.jl b/base/range.jl index af28228cd74ac..517d0456db5e8 100644 --- a/base/range.jl +++ b/base/range.jl @@ -608,8 +608,8 @@ function getindex(v::AbstractRange{T}, i::Integer) where T @_inline_meta ret = convert(T, first(v) + (i - 1)*step_hp(v)) ok = ifelse(step(v) > zero(step(v)), - (ret <= v.stop) & (ret >= v.start), - (ret <= v.start) & (ret >= v.stop)) + (ret <= last(v)) & (ret >= first(v)), + (ret <= first(v)) & (ret >= last(v))) @boundscheck ((i > 0) & ok) || throw_boundserror(v, i) ret end diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 680e94c57b7ec..27cbbe947bcbc 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -92,6 +92,10 @@ do33 = fill(1.,3) end end +@testset "Issue #30006" begin + SparseMatrixCSC{Float64,Int32}(spzeros(3,3))[:, 1] == [1, 2, 3] +end + @testset "concatenation tests" begin sp33 = sparse(1.0I, 3, 3) diff --git a/test/ranges.jl b/test/ranges.jl index cfb41a477ebef..dba2e8b24983c 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -1406,6 +1406,13 @@ end @test @inferred(z4 .+ z4) === z4 end +@testset "Issue #30006" begin + @test Base.Slice(Base.OneTo(5))[Int32(1)] == Int32(1) + @test Base.Slice(Base.OneTo(3))[Int8(2)] == Int8(2) + @test Base.Slice(1:10)[Int32(2)] == Int32(2) + @test Base.Slice(1:10)[Int8(2)] == Int8(2) +end + @testset "allocation of TwicePrecision call" begin 0:286.493442:360 0:286:360 From fb28ab4162c185c445cac02657d88c13561bb196 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 23 Nov 2018 02:57:46 -0800 Subject: [PATCH 21/41] Fix method ambiguities in SparseArrays (#30120) * Remove unused struct CapturedScalars * Fix method ambiguities in SparseArrays * Fix HigherOrderFns._copy(f) (cherry picked from commit f10530ef93af92fe11bc76738c2098a8cd638f49) --- stdlib/SparseArrays/src/higherorderfns.jl | 11 +++++----- stdlib/SparseArrays/test/ambiguous_exec.jl | 4 ++++ stdlib/SparseArrays/test/higherorderfns.jl | 24 ++++++++++++++++++++++ stdlib/SparseArrays/test/sparse.jl | 8 ++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 stdlib/SparseArrays/test/ambiguous_exec.jl diff --git a/stdlib/SparseArrays/src/higherorderfns.jl b/stdlib/SparseArrays/src/higherorderfns.jl index e09ca104c4220..63d318af434f9 100644 --- a/stdlib/SparseArrays/src/higherorderfns.jl +++ b/stdlib/SparseArrays/src/higherorderfns.jl @@ -973,6 +973,7 @@ function _copy(f, args...) parevalf, passedargstup = capturescalars(f, args) return _copy(parevalf, passedargstup...) end +_copy(f) = throw(MethodError(_copy, (f,))) # avoid method ambiguity function _shapecheckbc(f, args...) _aresameshape(args...) ? _noshapecheck_map(f, args...) : _diffshape_broadcast(f, args...) @@ -1006,10 +1007,6 @@ end _copyto!(parevalf, dest, passedsrcargstup...) end -struct CapturedScalars{F, Args, Order} - args::Args -end - # capturescalars takes a function (f) and a tuple of mixed sparse vectors/matrices and # broadcast scalar arguments (mixedargs), and returns a function (parevalf, i.e. partially # evaluated f) and a reduced argument tuple (passedargstup) containing only the sparse @@ -1024,9 +1021,13 @@ end # Work around losing Type{T}s as DataTypes within the tuple that makeargs creates @inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Vararg{Any}}) where {T} = capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) +@inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Ref{Type{S}}, Vararg{Any}}) where {T, S} = + # This definition is identical to the one above and necessary only for + # avoiding method ambiguity. + capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) @inline capturescalars(f, mixedargs::Tuple{SparseVecOrMat, Ref{Type{T}}, Vararg{Any}}) where {T} = capturescalars((a1, args...)->f(a1, T, args...), (mixedargs[1], Base.tail(Base.tail(mixedargs))...)) -@inline capturescalars(f, mixedargs::Tuple{Union{Ref,AbstractArray{0}}, Ref{Type{T}}, Vararg{Any}}) where {T} = +@inline capturescalars(f, mixedargs::Tuple{Union{Ref,AbstractArray{<:Any,0}}, Ref{Type{T}}, Vararg{Any}}) where {T} = capturescalars((args...)->f(mixedargs[1], T, args...), Base.tail(Base.tail(mixedargs))) nonscalararg(::SparseVecOrMat) = true diff --git a/stdlib/SparseArrays/test/ambiguous_exec.jl b/stdlib/SparseArrays/test/ambiguous_exec.jl new file mode 100644 index 0000000000000..a466f2534794a --- /dev/null +++ b/stdlib/SparseArrays/test/ambiguous_exec.jl @@ -0,0 +1,4 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, SparseArrays +@test detect_ambiguities(SparseArrays; imported=true, recursive=true) == [] diff --git a/stdlib/SparseArrays/test/higherorderfns.jl b/stdlib/SparseArrays/test/higherorderfns.jl index 5871e2d96e1e2..5b1423582f0d5 100644 --- a/stdlib/SparseArrays/test/higherorderfns.jl +++ b/stdlib/SparseArrays/test/higherorderfns.jl @@ -632,4 +632,28 @@ end @test minimum(sparse([1, 2], [1, 2], ones(Int32, 2)), dims = 1) isa Matrix end +@testset "Issue #30118" begin + @test ((_, x) -> x).(Int, spzeros(3)) == spzeros(3) + @test ((_, _, x) -> x).(Int, Int, spzeros(3)) == spzeros(3) + @test ((_, _, _, x) -> x).(Int, Int, Int, spzeros(3)) == spzeros(3) + @test_broken ((_, _, _, _, x) -> x).(Int, Int, Int, Int, spzeros(3)) == spzeros(3) +end + +using SparseArrays.HigherOrderFns: SparseVecStyle + +@testset "Issue #30120: method ambiguity" begin + # HigherOrderFns._copy(f) was ambiguous. It may be impossible to + # invoke this from dot notation and it is an error anyway. But + # when someone invokes it by accident, we want it to produce a + # meaningful error. + err = try + copy(Broadcast.Broadcasted{SparseVecStyle}(rand, ())) + catch err + err + end + @test err isa MethodError + @test !occursin("is ambiguous", sprint(showerror, err)) + @test occursin("no method matching _copy(::typeof(rand))", sprint(showerror, err)) +end + end # module diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 27cbbe947bcbc..d82fac08b6a1e 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -2341,4 +2341,12 @@ end @test m2.module == SparseArrays end +@testset "method ambiguity" begin + # Ambiguity test is run inside a clean process. + # https://github.com/JuliaLang/julia/issues/28804 + script = joinpath(@__DIR__, "ambiguous_exec.jl") + cmd = `$(Base.julia_cmd()) --startup-file=no $script` + @test success(pipeline(cmd; stdout=stdout, stderr=stderr)) +end + end # module From 7a7f552e902cba4bfafc4caf47cf1ccabd5e0031 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Mon, 17 Dec 2018 09:47:09 -0500 Subject: [PATCH 22/41] Fix #20409: rank for sparse matrices. (#30415) (cherry picked from commit f36ace9403f033a900f8c95c4fb79f256cac2a45) --- stdlib/SuiteSparse/src/spqr.jl | 1 + stdlib/SuiteSparse/test/spqr.jl | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/SuiteSparse/src/spqr.jl b/stdlib/SuiteSparse/src/spqr.jl index f851db416a714..44aad5c13956f 100644 --- a/stdlib/SuiteSparse/src/spqr.jl +++ b/stdlib/SuiteSparse/src/spqr.jl @@ -343,6 +343,7 @@ _ret_size(F::QRSparse, b::AbstractVector) = (size(F, 2),) _ret_size(F::QRSparse, B::AbstractMatrix) = (size(F, 2), size(B, 2)) LinearAlgebra.rank(F::QRSparse) = reduce(max, view(F.R.rowval, 1:nnz(F.R)), init = eltype(F.R.rowval)(0)) +LinearAlgebra.rank(S::SparseMatrixCSC) = rank(qr(S)) function (\)(F::QRSparse{T}, B::VecOrMat{Complex{T}}) where T<:LinearAlgebra.BlasReal # |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| diff --git a/stdlib/SuiteSparse/test/spqr.jl b/stdlib/SuiteSparse/test/spqr.jl index 96ee478937297..faabf336bde1a 100644 --- a/stdlib/SuiteSparse/test/spqr.jl +++ b/stdlib/SuiteSparse/test/spqr.jl @@ -95,8 +95,11 @@ end end @testset "rank" begin - @test rank(qr(sprandn(10, 5, 1.0)*sprandn(5, 10, 1.0))) == 5 + S = sprandn(10, 5, 1.0)*sprandn(5, 10, 1.0) + @test rank(qr(S)) == 5 + @test rank(S) == 5 @test all(iszero, (rank(qr(spzeros(10, i))) for i in 1:10)) + @test all(iszero, (rank(spzeros(10, i)) for i in 1:10)) end end From e3af2c12e943cf85bd92ae15259f270f80a8c0e0 Mon Sep 17 00:00:00 2001 From: Raghvendra Gupta Date: Mon, 17 Dec 2018 20:22:59 +0530 Subject: [PATCH 23/41] Fix sparse cholesky to return Vector when the RHS is a Vector (#30416) Fixes #28985 (cherry picked from commit b45100126d9f41f37dcb22e42d67f4cfa3ee9944) --- stdlib/SuiteSparse/src/cholmod.jl | 10 +++++++++- stdlib/SuiteSparse/test/cholmod.jl | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/stdlib/SuiteSparse/src/cholmod.jl b/stdlib/SuiteSparse/src/cholmod.jl index b0a82eaca18d6..f005e715b59d5 100644 --- a/stdlib/SuiteSparse/src/cholmod.jl +++ b/stdlib/SuiteSparse/src/cholmod.jl @@ -1717,10 +1717,18 @@ end (\)(L::Factor, B::SparseVecOrMat) = sparse(spsolve(CHOLMOD_A, L, Sparse(B, 0))) \(adjL::Adjoint{<:Any,<:Factor}, B::Dense) = (L = adjL.parent; solve(CHOLMOD_A, L, B)) -\(adjL::Adjoint{<:Any,<:Factor}, B::VecOrMat) = (L = adjL.parent; Matrix(solve(CHOLMOD_A, L, Dense(B)))) \(adjL::Adjoint{<:Any,<:Factor}, B::Sparse) = (L = adjL.parent; spsolve(CHOLMOD_A, L, B)) \(adjL::Adjoint{<:Any,<:Factor}, B::SparseVecOrMat) = (L = adjL.parent; \(adjoint(L), Sparse(B))) +function \(adjL::Adjoint{<:Any,<:Factor}, b::StridedVector) + L = adjL.parent + return Vector(solve(CHOLMOD_A, L, Dense(b))) +end +function \(adjL::Adjoint{<:Any,<:Factor}, B::StridedMatrix) + L = adjL.parent + return Matrix(solve(CHOLMOD_A, L, Dense(B))) +end + const RealHermSymComplexHermF64SSL = Union{ Symmetric{Float64,SparseMatrixCSC{Float64,SuiteSparse_long}}, Hermitian{Float64,SparseMatrixCSC{Float64,SuiteSparse_long}}, diff --git a/stdlib/SuiteSparse/test/cholmod.jl b/stdlib/SuiteSparse/test/cholmod.jl index aaa9e49266ca5..08dd32f4928b6 100644 --- a/stdlib/SuiteSparse/test/cholmod.jl +++ b/stdlib/SuiteSparse/test/cholmod.jl @@ -663,6 +663,11 @@ end @test_throws ArgumentError logdet(Fnew) end +@testset "Issue #28985" begin + @test typeof(cholesky(sparse(I, 4, 4))'\rand(4)) == Array{Float64, 1} + @test typeof(cholesky(sparse(I, 4, 4))'\rand(4,1)) == Array{Float64, 2} +end + @testset "Issue with promotion during conversion to CHOLMOD.Dense" begin @test CHOLMOD.Dense(fill(1, 5)) == fill(1, 5, 1) @test CHOLMOD.Dense(fill(1f0, 5)) == fill(1, 5, 1) From 7d63456fb3b3b5061b01cf91f0aa56bf3ee671d7 Mon Sep 17 00:00:00 2001 From: Klaus Crusius Date: Mon, 17 Dec 2018 16:58:35 +0100 Subject: [PATCH 24/41] spmatmul sparse matrix multiplication - performance improvements (#30372) * General performance improvements for sparse matmul Details for the polyalgorithm are in: https://github.com/JuliaLang/julia/pull/30372 (cherry picked from commit fae262c86111ae584d84d2bcf090a8026dbe95e3) --- stdlib/SparseArrays/src/linalg.jl | 97 +++++++++++++++++++++--------- stdlib/SparseArrays/test/sparse.jl | 3 +- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/stdlib/SparseArrays/src/linalg.jl b/stdlib/SparseArrays/src/linalg.jl index d6a97e759b8d9..44bb763272734 100644 --- a/stdlib/SparseArrays/src/linalg.jl +++ b/stdlib/SparseArrays/src/linalg.jl @@ -146,63 +146,104 @@ end *(A::Adjoint{<:Any,<:SparseMatrixCSC{Tv,Ti}}, B::Adjoint{<:Any,<:SparseMatrixCSC{Tv,Ti}}) where {Tv,Ti} = spmatmul(copy(A), copy(B)) *(A::Transpose{<:Any,<:SparseMatrixCSC{Tv,Ti}}, B::Transpose{<:Any,<:SparseMatrixCSC{Tv,Ti}}) where {Tv,Ti} = spmatmul(copy(A), copy(B)) -function spmatmul(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixCSC{Tv,Ti}; - sortindices::Symbol = :sortcols) where {Tv,Ti} +# Gustavsen's matrix multiplication algorithm revisited. +# The result rowval vector is already sorted by construction. +# The auxiliary Vector{Ti} xb is replaced by a Vector{Bool} of same length. +# The optional argument controlling a sorting algorithm is obsolete. +# depending on expected execution speed the sorting of the result column is +# done by a quicksort of the row indices or by a full scan of the dense result vector. +# The last is faster, if more than ≈ 1/32 of the result column is nonzero. +# TODO: extend to SparseMatrixCSCUnion to allow for SubArrays (view(X, :, r)). +function spmatmul(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixCSC{Tv,Ti}) where {Tv,Ti} mA, nA = size(A) - mB, nB = size(B) - nA==mB || throw(DimensionMismatch()) + nB = size(B, 2) + nA == size(B, 1) || throw(DimensionMismatch()) - colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval - colptrB = B.colptr; rowvalB = B.rowval; nzvalB = B.nzval - # TODO: Need better estimation of result space - nnzC = min(mA*nB, length(nzvalA) + length(nzvalB)) + rowvalA = rowvals(A); nzvalA = nonzeros(A) + rowvalB = rowvals(B); nzvalB = nonzeros(B) + nnzC = max(estimate_mulsize(mA, nnz(A), nA, nnz(B), nB) * 11 ÷ 10, mA) colptrC = Vector{Ti}(undef, nB+1) rowvalC = Vector{Ti}(undef, nnzC) nzvalC = Vector{Tv}(undef, nnzC) + nzpercol = nnzC ÷ max(nB, 1) @inbounds begin ip = 1 - xb = zeros(Ti, mA) - x = zeros(Tv, mA) + xb = fill(false, mA) for i in 1:nB if ip + mA - 1 > nnzC - resize!(rowvalC, nnzC + max(nnzC,mA)) - resize!(nzvalC, nnzC + max(nnzC,mA)) - nnzC = length(nzvalC) + nnzC += max(mA, nnzC>>2) + resize!(rowvalC, nnzC) + resize!(nzvalC, nnzC) end - colptrC[i] = ip - for jp in colptrB[i]:(colptrB[i+1] - 1) + colptrC[i] = ip0 = ip + k0 = ip - 1 + for jp in nzrange(B, i) nzB = nzvalB[jp] j = rowvalB[jp] - for kp in colptrA[j]:(colptrA[j+1] - 1) + for kp in nzrange(A, j) nzC = nzvalA[kp] * nzB k = rowvalA[kp] - if xb[k] != i + if xb[k] + nzvalC[k+k0] += nzC + else + nzvalC[k+k0] = nzC + xb[k] = true rowvalC[ip] = k ip += 1 - xb[k] = i - x[k] = nzC - else - x[k] += nzC end end end - for vp in colptrC[i]:(ip - 1) - nzvalC[vp] = x[rowvalC[vp]] + if ip > ip0 + if prefer_sort(ip-k0, mA) + # in-place sort of indices. Effort: O(nnz*ln(nnz)). + sort!(rowvalC, ip0, ip-1, QuickSort, Base.Order.Forward) + for vp = ip0:ip-1 + k = rowvalC[vp] + xb[k] = false + nzvalC[vp] = nzvalC[k+k0] + end + else + # scan result vector (effort O(mA)) + for k = 1:mA + if xb[k] + xb[k] = false + rowvalC[ip0] = k + nzvalC[ip0] = nzvalC[k+k0] + ip0 += 1 + end + end + end end end colptrC[nB+1] = ip end - deleteat!(rowvalC, colptrC[end]:length(rowvalC)) - deleteat!(nzvalC, colptrC[end]:length(nzvalC)) + resize!(rowvalC, ip - 1) + resize!(nzvalC, ip - 1) - # The Gustavson algorithm does not guarantee the product to have sorted row indices. - Cunsorted = SparseMatrixCSC(mA, nB, colptrC, rowvalC, nzvalC) - C = SparseArrays.sortSparseMatrixCSC!(Cunsorted, sortindices=sortindices) + # This modification of Gustavson algorithm has sorted row indices + C = SparseMatrixCSC(mA, nB, colptrC, rowvalC, nzvalC) return C end +# estimated number of non-zeros in matrix product +# it is assumed, that the non-zero indices are distributed independently and uniformly +# in both matrices. Over-estimation is possible if that is not the case. +function estimate_mulsize(m::Integer, nnzA::Integer, n::Integer, nnzB::Integer, k::Integer) + p = (nnzA / (m * n)) * (nnzB / (n * k)) + p >= 1 ? m*k : p > 0 ? Int(ceil(-expm1(log1p(-p) * n)*m*k)) : 0 # (1-(1-p)^n)*m*k +end + +# determine if sort! shall be used or the whole column be scanned +# based on empirical data on i7-3610QM CPU +# measuring runtimes of the scanning and sorting loops of the algorithm. +# The parameters 6 and 3 might be modified for different architectures. +prefer_sort(nz::Integer, m::Integer) = m > 6 && 3 * ilog2(nz) * nz < m + +# minimal number of bits required to represent integer; ilog2(n) >= log2(n) +ilog2(n::Integer) = sizeof(n)<<3 - leading_zeros(n) + # Frobenius dot/inner product: trace(A'B) function dot(A::SparseMatrixCSC{T1,S1},B::SparseMatrixCSC{T2,S2}) where {T1,T2,S1,S2} m, n = size(A) diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index d82fac08b6a1e..3447c56eca541 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -325,8 +325,7 @@ end a = sprand(10, 5, 0.7) b = sprand(5, 15, 0.3) @test maximum(abs.(a*b - Array(a)*Array(b))) < 100*eps() - @test maximum(abs.(SparseArrays.spmatmul(a,b,sortindices=:sortcols) - Array(a)*Array(b))) < 100*eps() - @test maximum(abs.(SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - Array(a)*Array(b))) < 100*eps() + @test maximum(abs.(SparseArrays.spmatmul(a,b) - Array(a)*Array(b))) < 100*eps() f = Diagonal(rand(5)) @test Array(a*f) == Array(a)*f @test Array(f*b) == f*Array(b) From 8c2ef2571d4fe975e1b831c7005be6e186f0e635 Mon Sep 17 00:00:00 2001 From: Raghvendra Gupta Date: Thu, 27 Dec 2018 12:13:32 +0530 Subject: [PATCH 25/41] Fixed sprand for dimensions of unusual integer type (#30516) Fixes #30502 (cherry picked from commit 2ab1405c9c3522f3dd5dcd96342e9681855ae78d) --- stdlib/SparseArrays/src/sparsematrix.jl | 2 ++ stdlib/SparseArrays/test/sparse.jl | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/stdlib/SparseArrays/src/sparsematrix.jl b/stdlib/SparseArrays/src/sparsematrix.jl index 165d2d6a53685..6b5c9b2585375 100644 --- a/stdlib/SparseArrays/src/sparsematrix.jl +++ b/stdlib/SparseArrays/src/sparsematrix.jl @@ -1443,6 +1443,7 @@ julia> sprand(Float64, 3, 0.75) """ function sprand(r::AbstractRNG, m::Integer, n::Integer, density::AbstractFloat, rfn::Function, ::Type{T}=eltype(rfn(r,1))) where T + m,n = Int(m), Int(n) N = m*n N == 0 && return spzeros(T,m,n) N == 1 && return rand(r) <= density ? sparse([1], [1], rfn(r,1)) : spzeros(T,1,1) @@ -1453,6 +1454,7 @@ end function sprand(m::Integer, n::Integer, density::AbstractFloat, rfn::Function, ::Type{T}=eltype(rfn(1))) where T + m,n = Int(m), Int(n) N = m*n N == 0 && return spzeros(T,m,n) N == 1 && return rand() <= density ? sparse([1], [1], rfn(1)) : spzeros(T,1,1) diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 3447c56eca541..163f3b9271f1b 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -332,6 +332,11 @@ end end end +@testset "Issue #30502" begin + @test nnz(sprand(UInt8(16), UInt8(16), 1.0)) == 256 + @test nnz(sprand(UInt8(16), UInt8(16), 1.0, ones)) == 256 +end + @testset "kronecker product" begin for (m,n) in ((5,10), (13,8), (14,10)) a = sprand(m, 5, 0.4); a_d = Matrix(a) From 7542c72da25199f3cbf075b62ea05d7110cd4579 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 2 Jan 2019 21:06:42 +0100 Subject: [PATCH 26/41] Add custom deserialize method for UmfpackLU to avoid memory leak (#30425) Fixes #15450 (cherry picked from commit 356ceeee0170e41ac109abc0841d394f5472c5e7) --- stdlib/SuiteSparse/Project.toml | 5 +++-- stdlib/SuiteSparse/src/umfpack.jl | 18 ++++++++++++++++++ stdlib/SuiteSparse/test/umfpack.jl | 13 +++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/stdlib/SuiteSparse/Project.toml b/stdlib/SuiteSparse/Project.toml index 8ea21106262c4..a851e081a383e 100644 --- a/stdlib/SuiteSparse/Project.toml +++ b/stdlib/SuiteSparse/Project.toml @@ -4,13 +4,14 @@ uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Random", "DelimitedFiles", "Serialization"] diff --git a/stdlib/SuiteSparse/src/umfpack.jl b/stdlib/SuiteSparse/src/umfpack.jl index a8eb90ed379c4..23c0708fb72d8 100644 --- a/stdlib/SuiteSparse/src/umfpack.jl +++ b/stdlib/SuiteSparse/src/umfpack.jl @@ -11,6 +11,8 @@ import LinearAlgebra: Factorization, det, lu, ldiv! using SparseArrays import SparseArrays: nnz +import Serialization: AbstractSerializer, deserialize + import ..increment, ..increment!, ..decrement, ..decrement! include("umfpack_h.jl") @@ -192,6 +194,22 @@ function show(io::IO, F::UmfpackLU) F.numeric != C_NULL && print(io, '\n', F.numeric) end +function deserialize(s::AbstractSerializer, t::Type{UmfpackLU{Tv,Ti}}) where {Tv,Ti} + symbolic = deserialize(s) + numeric = deserialize(s) + m = deserialize(s) + n = deserialize(s) + colptr = deserialize(s) + rowval = deserialize(s) + nzval = deserialize(s) + status = deserialize(s) + obj = UmfpackLU{Tv,Ti}(symbolic, numeric, m, n, colptr, rowval, nzval, status) + + finalizer(umfpack_free_symbolic, obj) + + return obj +end + ## Wrappers for UMFPACK functions # generate the name of the C function according to the value and integer types diff --git a/stdlib/SuiteSparse/test/umfpack.jl b/stdlib/SuiteSparse/test/umfpack.jl index 1683f21aa4aab..12ce4c9bddac4 100644 --- a/stdlib/SuiteSparse/test/umfpack.jl +++ b/stdlib/SuiteSparse/test/umfpack.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using SuiteSparse: increment! +using Serialization using LinearAlgebra: Adjoint, Transpose, SingularException @testset "UMFPACK wrappers" begin @@ -176,4 +177,16 @@ using LinearAlgebra: Adjoint, Transpose, SingularException end end + @testset "deserialization" begin + A = sprandn(10, 10, 0.4) + F1 = lu(A) + b = IOBuffer() + serialize(b, F1) + seekstart(b) + F2 = deserialize(b) + for nm in (:colptr, :m, :n, :nzval, :rowval, :status) + @test getfield(F1, nm) == getfield(F2, nm) + end + end + end From bb23a90ae951ab55c94befcf93d8768350e61c55 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 21 Dec 2018 12:35:30 +0100 Subject: [PATCH 27/41] Fix performance of broadcast and collect with Union{T, Missing} Use the same pattern as in collect_to_with_first! (which is used when size is known). (cherry picked from commit 184fbc48a82e3c2b7ba52c97fe2774a255928b6a) --- base/array.jl | 3 +-- base/broadcast.jl | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/base/array.jl b/base/array.jl index dcea65bcadba6..67865130022c6 100644 --- a/base/array.jl +++ b/base/array.jl @@ -685,8 +685,7 @@ function grow_to!(dest, itr, st) y = iterate(itr, st) while y !== nothing el, st = y - S = typeof(el) - if S === T || S <: T + if el isa T || typeof(el) === T push!(dest, el::T) else new = sizehint!(empty(dest, promote_typejoin(T, S)), length(dest)) diff --git a/base/broadcast.jl b/base/broadcast.jl index db85efd85c281..f0ec04e8de1bc 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -926,13 +926,12 @@ function copyto_nonleaf!(dest, bc::Broadcasted, iter, state, count) y === nothing && break I, state = y @inbounds val = bc[I] - S = typeof(val) - if S <: T + if val isa T || typeof(val) === T @inbounds dest[I] = val else # This element type doesn't fit in dest. Allocate a new dest with wider eltype, # copy over old values, and continue - newdest = Base.similar(dest, promote_typejoin(T, S)) + newdest = Base.similar(dest, promote_typejoin(T, typeof(val))) for II in Iterators.take(iter, count) newdest[II] = dest[II] end From 7bbc99ab5ac989c9b3cc88bc5937c4f7f2716f22 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 3 Jan 2019 13:14:55 -0500 Subject: [PATCH 28/41] gf: fix jl_isa_compileable_sig again (#30458) The last attempts were pretty good, but still missed a lot. But this is what you find when you actually try to test it. This is not too important, but it can reduce compilation performance in some case, so it is not ideal. (cherry picked from commit 76e7421cda22d64d0c7d67ebdf4043fcb526b325) --- src/codegen.cpp | 2 +- src/gf.c | 168 +++++++++++++++++++++++++++++++----------------- test/core.jl | 4 +- 3 files changed, 111 insertions(+), 63 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 89a9727755155..5e0e5a2ec8a62 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4209,7 +4209,7 @@ static void emit_cfunc_invalidate( case jl_returninfo_t::Union: { Type *retty = gf_thunk->getReturnType(); Value *gf_retval = UndefValue::get(retty); - Value *tindex = compute_box_tindex(ctx, gf_ret, (jl_value_t*)jl_any_type, astrt); + Value *tindex = compute_box_tindex(ctx, emit_typeof_boxed(ctx, gf_retbox), (jl_value_t*)jl_any_type, astrt); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); gf_retval = ctx.builder.CreateInsertValue(gf_retval, gf_ret, 0); gf_retval = ctx.builder.CreateInsertValue(gf_retval, tindex, 1); diff --git a/src/gf.c b/src/gf.c index 520db1ab45a98..f79309ca0e39d 100644 --- a/src/gf.c +++ b/src/gf.c @@ -626,6 +626,17 @@ static void jl_compilation_sig( jl_svecset(*newparams, i, elt); } } + else if (jl_is_kind(elt)) { + // not triggered for isdispatchtuple(tt), this attempts to handle + // some cases of adapting a random signature into a compilation signature + // if we get a kind, where we don't expect to accept one, widen it to something more expected (Type{T}) + if (!(jl_subtype(elt, decl_i) && !jl_subtype((jl_value_t*)jl_type_type, decl_i))) { + if (!*newparams) *newparams = jl_svec_copy(tt->parameters); + elt = (jl_value_t*)jl_typetype_type; + jl_svecset(*newparams, i, elt); + } + } + if (jl_is_kind(elt)) { // kind slots always need guard entries (checking for subtypes of Type) @@ -643,7 +654,17 @@ static void jl_compilation_sig( } } - if (jl_is_type_type(elt)) { + if (jl_types_equal(elt, (jl_value_t*)jl_typetype_type)) { + // not triggered for isdispatchtuple(tt), this attempts to handle + // some cases of adapting a random signature into a compilation signature + } + else if (!jl_is_datatype(elt) && !jl_has_empty_intersection((jl_value_t*)jl_type_type, elt)) { + // not triggered for isdispatchtuple(tt), this attempts to handle + // some cases of adapting a random signature into a compilation signature + if (!*newparams) *newparams = jl_svec_copy(tt->parameters); + jl_svecset(*newparams, i, jl_typetype_type); + } + else if (jl_is_type_type(elt)) { if (very_general_type(decl_i)) { /* here's a fairly simple heuristic: if this argument slot's @@ -707,7 +728,6 @@ static void jl_compilation_sig( // when called with a subtype of Function but is not called if (!*newparams) *newparams = jl_svec_copy(tt->parameters); jl_svecset(*newparams, i, (jl_value_t*)jl_function_type); - continue; } } @@ -715,10 +735,9 @@ static void jl_compilation_sig( // in general, here we want to find the biggest type that's not a // supertype of any other method signatures. so far we are conservative // and the types we find should be bigger. - if (jl_nparams(tt) >= nspec && jl_va_tuple_kind((jl_datatype_t*)definition->sig) == JL_VARARG_UNBOUND) { + if (jl_nparams(tt) >= nspec && jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND) { jl_svec_t *limited = jl_alloc_svec(nspec); - jl_value_t *temp = NULL; - JL_GC_PUSH2(&limited, &temp); + JL_GC_PUSH1(&limited); if (!*newparams) *newparams = tt->parameters; size_t i; for (i = 0; i < nspec - 1; i++) { @@ -742,27 +761,21 @@ static void jl_compilation_sig( } } if (all_are_subtypes) { - // avoid Type{Type{...}}... + // avoid Vararg{Type{Type{...}}} if (jl_is_type_type(lasttype) && jl_is_type_type(jl_tparam0(lasttype))) lasttype = (jl_value_t*)jl_type_type; jl_svecset(limited, i, jl_wrap_vararg(lasttype, (jl_value_t*)NULL)); } else { - jl_value_t *unw = jl_unwrap_unionall(definition->sig); - jl_value_t *lastdeclt = jl_tparam(unw, jl_nparams(unw) - 1); + jl_value_t *unw = jl_unwrap_unionall(decl); + jl_value_t *lastdeclt = jl_tparam(unw, nargs - 1); + assert(jl_is_vararg_type(lastdeclt) && jl_nparams(unw) == nargs); int nsp = jl_svec_len(sparams); - if (nsp > 0) { - jl_svec_t *env = jl_alloc_svec_uninit(2 * nsp); - temp = (jl_value_t*)env; - jl_unionall_t *ua = (jl_unionall_t*)definition->sig; - for (j = 0; j < nsp; j++) { - assert(jl_is_unionall(ua)); - jl_svecset(env, j * 2, ua->var); - jl_svecset(env, j * 2 + 1, jl_svecref(sparams, j)); - ua = (jl_unionall_t*)ua->body; - } - lastdeclt = (jl_value_t*)jl_instantiate_type_with((jl_value_t*)lastdeclt, - jl_svec_data(env), nsp); + if (nsp > 0 && jl_has_free_typevars(lastdeclt)) { + assert(jl_subtype_env_size(decl) == nsp); + lastdeclt = jl_instantiate_type_in_env(lastdeclt, (jl_unionall_t*)decl, jl_svec_data(sparams)); + // TODO: rewrap_unionall(lastdeclt, sparams) if any sparams isa TypeVar??? + // TODO: if we made any replacements above, sparams may now be incorrect } jl_svecset(limited, i, lastdeclt); } @@ -794,18 +807,40 @@ JL_DLLEXPORT int jl_isa_compileable_sig( size_t nargs = definition->nargs; // == jl_field_count(jl_unwrap_unionall(decl)); if (np == 0) return nargs == 0; - if (jl_is_vararg_type(jl_tparam(type, np - 1))) { - if (!definition->isva || np <= nargs) - return 0; - } - else if (definition->isva ? np != nargs : np < nargs) { - return 0; - } if (definition->generator) { // staged functions aren't optimized // so assume the caller was intelligent about calling us - return type->isdispatchtuple; + return (definition->isva ? np >= nargs - 1 : np == nargs) && type->isdispatchtuple; + } + + // for varargs methods, only specialize up to max_args (>= nargs + 1). + // in general, here we want to find the biggest type that's not a + // supertype of any other method signatures. so far we are conservative + // and the types we find should be bigger. + if (definition->isva) { + unsigned nspec_min = nargs + 1; // min number of non-vararg values before vararg + unsigned nspec_max = INT32_MAX; // max number of non-vararg values before vararg + jl_datatype_t *gf = jl_first_argument_datatype(decl); + if (gf != NULL && jl_is_datatype(gf) && gf->name->mt != NULL) { + // try to refine estimate of min and max + if (gf->name->mt != jl_type_type_mt) + nspec_min = gf->name->mt->max_args + 2; + else + nspec_max = nspec_min; + } + int isbound = (jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND); + if (jl_is_vararg_type(jl_tparam(type, np - 1))) { + if (!isbound || np < nspec_min || np > nspec_max) + return 0; + } + else { + if (np < nargs - 1 || (isbound && np >= nspec_max)) + return 0; + } + } + else if (np != nargs || jl_is_vararg_type(jl_tparam(type, np - 1))) { + return 0; } for (i = 0; i < np; i++) { @@ -813,10 +848,24 @@ JL_DLLEXPORT int jl_isa_compileable_sig( jl_value_t *decl_i = jl_nth_slot_type((jl_value_t*)decl, i); size_t i_arg = (i < nargs - 1 ? i : nargs - 1); - if (jl_is_vararg_type(elt)) { // varargs are always considered compilable - if (!jl_has_free_typevars(elt)) + if (jl_is_vararg_type(elt)) { + elt = jl_unwrap_vararg(elt); + if (jl_has_free_typevars(decl_i)) { + // TODO: in this case, answer semi-conservatively that these varargs are always compilable + // we don't have the ability to get sparams, so deciding if elt + // is a potential result of jl_instantiate_type_in_env for decl_i + // for any sparams that is consistent with the rest of the arguments + // seems like it would be extremely difficult + // and hopefully the upstream code probably gave us something reasonable continue; - return 0; + } + else if (jl_egal(elt, decl_i)) { + continue; + } + else if (jl_is_type_type(elt) && jl_is_type_type(jl_tparam0(elt))) { + return 0; + } + // else, it needs to meet the usual rules } if (i_arg > 0 && i_arg <= sizeof(definition->nospecialize) * 8 && @@ -830,44 +879,38 @@ JL_DLLEXPORT int jl_isa_compileable_sig( if (jl_is_kind(elt)) { // kind slots always get guard entries (checking for subtypes of Type) - if (decl_i == elt || jl_subtype((jl_value_t*)jl_type_type, decl_i)) + if (jl_subtype(elt, decl_i) && !jl_subtype((jl_value_t*)jl_type_type, decl_i)) continue; // TODO: other code paths that could reach here return 0; } + else if (jl_is_kind(decl_i)) { + return 0; + } + + if (jl_is_type_type(jl_unwrap_unionall(elt))) { + if (jl_types_equal(elt, (jl_value_t*)jl_type_type)) { + if (very_general_type(decl_i)) + continue; + if (i >= nargs && definition->isva) + continue; + return 0; + } + if (very_general_type(decl_i)) + return 0; + if (!jl_is_datatype(elt)) + return 0; - if (jl_is_type_type(elt)) { // if the declared type was not Any or Union{Type, ...}, // then the match must been with kind, such as UnionAll or DataType, // and the result of matching the type signature - // needs to be corrected to the concrete type 'kind' + // needs to be corrected to the concrete type 'kind' (and not to Type) jl_value_t *kind = jl_typeof(jl_tparam0(elt)); - if (kind != (jl_value_t*)jl_tvar_type && jl_subtype(kind, decl_i)) { - if (!jl_subtype((jl_value_t*)jl_type_type, decl_i)) - return 0; - } + if (kind == jl_bottom_type) + return 0; // Type{Union{}} gets normalized to typeof(Union{}) + if (jl_subtype(kind, decl_i) && !jl_subtype((jl_value_t*)jl_type_type, decl_i)) + return 0; // gets turned into a kind - if (very_general_type(decl_i)) { - /* - here's a fairly simple heuristic: if this argument slot's - declared type is general (Type or Any), - then don't specialize for every Type that got passed. - - Since every type x has its own type Type{x}, this would be - excessive specialization for an Any slot. - - This may require guard entries due to other potential matches. - In particular, TypeConstructors are problematic because they can - be alternate representations of any type. Extensionally, TC == TC.body, - but typeof(TC) != typeof(TC.body). This creates an ambiguity: - Type{TC} is type-equal to Type{TC.body}, yet a slot - x::TypeConstructor matches the first but not the second, while - also matching all other TypeConstructors. This means neither - Type{TC} nor TypeConstructor is more specific. - */ - if (elt != (jl_value_t*)jl_typetype_type) - return 0; - } else if (jl_is_type_type(jl_tparam0(elt)) && // give up on specializing static parameters for Type{Type{Type{...}}} (jl_is_type_type(jl_tparam0(jl_tparam0(elt))) || !jl_has_free_typevars(decl_i))) { @@ -886,7 +929,7 @@ JL_DLLEXPORT int jl_isa_compileable_sig( JL_GC_POP(); return 0; } - else if (!jl_subtype(di, elt) || !jl_subtype(elt, di)) { + else if (!jl_types_equal(di, elt)) { JL_GC_POP(); return 0; } @@ -950,7 +993,11 @@ static jl_method_instance_t *cache_method( cache_with_orig = 0; compilationsig = jl_apply_tuple_type(newparams); temp2 = (jl_value_t*)compilationsig; + // In most cases `!jl_isa_compileable_sig(tt, definition))`, + // although for some cases, (notably Varargs) + // we might choose a replacement type that's preferable but not strictly better } + // TODO: maybe assert(jl_isa_compileable_sig(compilationsig, definition)); newmeth = jl_specializations_get_linfo(definition, (jl_value_t*)compilationsig, sparams, world); jl_tupletype_t *cachett = tt; @@ -959,6 +1006,7 @@ static jl_method_instance_t *cache_method( size_t max_valid = definition->max_world; if (!cache_with_orig) { // now examine what will happen if we chose to use this sig in the cache + // TODO: should we first check `compilationsig <: definition`? temp = ml_matches(mt->defs, 0, compilationsig, -1, 0, world, &min_valid, &max_valid); // TODO: use MAX_UNSPECIALIZED_CONFLICTS? int guards = 0; if (temp == jl_false) { diff --git a/test/core.jl b/test/core.jl index ae9a90b5149ce..1ae09dfd438de 100644 --- a/test/core.jl +++ b/test/core.jl @@ -6421,8 +6421,8 @@ end # issue #21004 const PTuple_21004{N,T} = NTuple{N,VecElement{T}} -@test_throws ArgumentError PTuple_21004(1) -@test_throws UndefVarError PTuple_21004_2{N,T} = NTuple{N, VecElement{T}}(1) +@test_throws ArgumentError("too few elements for tuple type $PTuple_21004") PTuple_21004(1) +@test_throws UndefVarError(:T) PTuple_21004_2{N,T} = NTuple{N, VecElement{T}}(1) #issue #22792 foo_22792(::Type{<:Union{Int8,Int,UInt}}) = 1; From 0bd12d026620764157dc304e8c8782d751eaa344 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 3 Jan 2019 18:09:38 -0500 Subject: [PATCH 29/41] fix `lambda-optimize-vars!` with complex assignment RHSs (#30564) fixes #30563 (cherry picked from commit 84a83ab8219e48c2ed466fe5c76b3de74039c158) --- src/julia-syntax.scm | 4 +++- test/compiler/compiler.jl | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 5d9e99a9627d9..7b1a15c559103 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2998,7 +2998,9 @@ f(x) = yt(x) (kill)) (cdr e))) (else - (mark-used e) + (if (eq? (car e) '=) + (visit (caddr e)) + (mark-used e)) (if (and (or (eq? (car e) '=) (and (eq? (car e) 'method) (length> e 2))) (has? unused (cadr e))) diff --git a/test/compiler/compiler.jl b/test/compiler/compiler.jl index 8ff37af09aac9..791c73a61f7db 100644 --- a/test/compiler/compiler.jl +++ b/test/compiler/compiler.jl @@ -1822,6 +1822,15 @@ function g15276() end @test g15276() isa Vector{Int} +function inbounds_30563() + local y + @inbounds for i in 1:10 + y = (m->2i)(0) + end + return y +end +@test Base.return_types(inbounds_30563, ()) == Any[Int] + # issue #27316 - inference shouldn't hang on these f27316(::Vector) = nothing f27316(::Any) = f27316(Any[][1]), f27316(Any[][1]) From d7ad4bd05ff33766cd3564b50204f812502a1079 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Fri, 4 Jan 2019 00:05:38 -0500 Subject: [PATCH 30/41] Add the scaled identity matrix to a random matrix to avoid getting a singular matrix (#30576) Fix #30572 (cherry picked from commit 4d8a968a87d7edd4586ee901dffa4eaa6856b966) --- stdlib/SuiteSparse/test/umfpack.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/SuiteSparse/test/umfpack.jl b/stdlib/SuiteSparse/test/umfpack.jl index 12ce4c9bddac4..356c0cf3fa4e7 100644 --- a/stdlib/SuiteSparse/test/umfpack.jl +++ b/stdlib/SuiteSparse/test/umfpack.jl @@ -178,7 +178,7 @@ using LinearAlgebra: Adjoint, Transpose, SingularException end @testset "deserialization" begin - A = sprandn(10, 10, 0.4) + A = 10*I + sprandn(10, 10, 0.4) F1 = lu(A) b = IOBuffer() serialize(b, F1) From dc6994dd1ac083174db98563dfaf08320eaac3fd Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Fri, 4 Jan 2019 22:45:30 -0800 Subject: [PATCH 31/41] Use XCode 8.3 for macOS on Travis (#30599) This corresponds to macOS 10.12 Sierra. XCode 8 covers El Capitan and Sierra, so if Travis is giving us XCode 8.x for x < 3, we're on El Cap. Homebrew supports only three versions of macOS at a time, which means that El Cap (10.11) is no longer support. This is likely why our Mac builds are trying to build GCC from source; a bottle might not be available. (cherry picked from commit 862fe08de4283346b51e8dcddb3df52a5315ac6a) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b227c5832dc5f..6bbca3bc3a52d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ matrix: - gfortran-5 - os: osx env: ARCH="x86_64" - osx_image: xcode8 + osx_image: xcode8.3 cache: ccache branches: only: From 90ba684520ae8f9bb1b556eec242ce450d1c06fa Mon Sep 17 00:00:00 2001 From: Ashley Sommer Date: Sun, 6 Jan 2019 14:06:38 +1000 Subject: [PATCH 32/41] fix typo in string search api docs (#30600) (cherry picked from commit a25945a7c0a52a6853e51f9cd617cd053ff68476) --- base/strings/search.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/strings/search.jl b/base/strings/search.jl index cb819b3007177..8b62b34bc2d07 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -259,7 +259,7 @@ findnext(t::AbstractString, s::AbstractString, i::Integer) = _search(s, t, i) findlast(pattern::AbstractString, string::AbstractString) Find the last occurrence of `pattern` in `string`. Equivalent to -[`findlast(pattern, string, lastindex(s))`](@ref). +[`findprev(pattern, string, lastindex(string))`](@ref). # Examples ```jldoctest From d839fe0bb9e92bdd05864a18cd171098e845d791 Mon Sep 17 00:00:00 2001 From: petershintech <46059512+petershintech@users.noreply.github.com> Date: Mon, 7 Jan 2019 07:39:28 +1100 Subject: [PATCH 33/41] Make sure to call jl_islayout_inline() even in the case assertion is deactivated (#30615) (cherry picked from commit 293107265aa1d0e2bbb527c910775640f713b9b2) --- base/array.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 67865130022c6..9d213977e03b2 100644 --- a/base/array.jl +++ b/base/array.jl @@ -192,7 +192,8 @@ julia> Base.bitsunionsize(Union{Float64, UInt8, Int128}) function bitsunionsize(u::Union) sz = Ref{Csize_t}(0) algn = Ref{Csize_t}(0) - @assert ccall(:jl_islayout_inline, Cint, (Any, Ptr{Csize_t}, Ptr{Csize_t}), u, sz, algn) != Cint(0) + isunboxed = ccall(:jl_islayout_inline, Cint, (Any, Ptr{Csize_t}, Ptr{Csize_t}), u, sz, algn) + @assert isunboxed != Cint(0) return sz[] end From 8ee59bc3430de10b86a5c5ce0e2d553be0bc90d0 Mon Sep 17 00:00:00 2001 From: Alfredo Braunstein Date: Tue, 25 Dec 2018 14:06:40 +0100 Subject: [PATCH 34/41] faster circshift! for SparseMatrixCSC (#30317) * implement circshift! for SparseMatrixCSC * factor helper function shifter!, implement efficient circshift! for SparseVector * add some @inbounds for improved performance * remove allocations completely, giving a large improvement for small matrices * some renaming to avoid polluting the module namespace * remove useless reallocation and fix bug with different in/out types, better tests * avoid action if iszero(r) and/or iszero(c), move sparse vector shifting helpers to sparsevector.jl * Make shift amounts deterministic in tests, move sparse vector tests into sparsevector.jl * comment fix * for some reason, copy!(a::SparseVector, b::SparseVector) does not work (cherry picked from commit 94993e910b25f0cdd02f1c2e2d0dbd5695f43b6f) --- stdlib/SparseArrays/src/SparseArrays.jl | 2 +- stdlib/SparseArrays/src/sparsematrix.jl | 43 ++++++++++++++++++++++++ stdlib/SparseArrays/src/sparsevector.jl | 39 +++++++++++++++++++++ stdlib/SparseArrays/test/sparse.jl | 27 +++++++++++++++ stdlib/SparseArrays/test/sparsevector.jl | 22 ++++++++++++ 5 files changed, 132 insertions(+), 1 deletion(-) diff --git a/stdlib/SparseArrays/src/SparseArrays.jl b/stdlib/SparseArrays/src/SparseArrays.jl index 724b9865782a6..e122f9fa97790 100644 --- a/stdlib/SparseArrays/src/SparseArrays.jl +++ b/stdlib/SparseArrays/src/SparseArrays.jl @@ -27,7 +27,7 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, vcat, hcat, hvcat, cat, imag, argmax, kron, length, log, log1p, max, min, maximum, minimum, one, promote_eltype, real, reshape, rot180, rotl90, rotr90, round, setindex!, similar, size, transpose, - vec, permute!, map, map!, Array, diff + vec, permute!, map, map!, Array, diff, circshift!, circshift using Random: GLOBAL_RNG, AbstractRNG, randsubseq, randsubseq! diff --git a/stdlib/SparseArrays/src/sparsematrix.jl b/stdlib/SparseArrays/src/sparsematrix.jl index 6b5c9b2585375..6a7ef13b507c8 100644 --- a/stdlib/SparseArrays/src/sparsematrix.jl +++ b/stdlib/SparseArrays/src/sparsematrix.jl @@ -3503,3 +3503,46 @@ end (+)(A::SparseMatrixCSC, J::UniformScaling) = A + sparse(J, size(A)...) (-)(A::SparseMatrixCSC, J::UniformScaling) = A - sparse(J, size(A)...) (-)(J::UniformScaling, A::SparseMatrixCSC) = sparse(J, size(A)...) - A + +## circular shift + +function circshift!(O::SparseMatrixCSC, X::SparseMatrixCSC, (r,c)::Base.DimsInteger{2}) + nnz = length(X.nzval) + + iszero(nnz) && return copy!(O, X) + + ##### column shift + c = mod(c, X.n) + if iszero(c) + copy!(O, X) + else + ##### readjust output + resize!(O.colptr, X.n + 1) + resize!(O.rowval, nnz) + resize!(O.nzval, nnz) + O.colptr[X.n + 1] = nnz + 1 + + # exchange left and right blocks + nleft = X.colptr[X.n - c + 1] - 1 + nright = nnz - nleft + @inbounds for i=c+1:X.n + O.colptr[i] = X.colptr[i-c] + nright + end + @inbounds for i=1:c + O.colptr[i] = X.colptr[X.n - c + i] - nleft + end + # rotate rowval and nzval by the right number of elements + circshift!(O.rowval, X.rowval, (nright,)) + circshift!(O.nzval, X.nzval, (nright,)) + end + ##### row shift + r = mod(r, X.m) + iszero(r) && return O + @inbounds for i=1:O.n + subvector_shifter!(O.rowval, O.nzval, O.colptr[i], O.colptr[i+1]-1, O.m, r) + end + return O +end + +circshift!(O::SparseMatrixCSC, X::SparseMatrixCSC, (r,)::Base.DimsInteger{1}) = circshift!(O, X, (r,0)) +circshift!(O::SparseMatrixCSC, X::SparseMatrixCSC, r::Real) = circshift!(O, X, (Integer(r),0)) diff --git a/stdlib/SparseArrays/src/sparsevector.jl b/stdlib/SparseArrays/src/sparsevector.jl index e7f0e1a2db26a..12c90991e78f2 100644 --- a/stdlib/SparseArrays/src/sparsevector.jl +++ b/stdlib/SparseArrays/src/sparsevector.jl @@ -1974,3 +1974,42 @@ function fill!(A::Union{SparseVector, SparseMatrixCSC}, x) end return A end + + + +# in-place swaps (dense) blocks start:split and split+1:fin in col +function _swap!(col::AbstractVector, start::Integer, fin::Integer, split::Integer) + split == fin && return + reverse!(col, start, split) + reverse!(col, split + 1, fin) + reverse!(col, start, fin) + return +end + + +# in-place shifts a sparse subvector by r. Used also by sparsematrix.jl +function subvector_shifter!(R::AbstractVector, V::AbstractVector, start::Integer, fin::Integer, m::Integer, r::Integer) + split = fin + @inbounds for j = start:fin + # shift positions ... + R[j] += r + if R[j] <= m + split = j + else + R[j] -= m + end + end + # ...but rowval should be sorted within columns + _swap!(R, start, fin, split) + _swap!(V, start, fin, split) +end + + +function circshift!(O::SparseVector, X::SparseVector, (r,)::Base.DimsInteger{1}) + O .= X + subvector_shifter!(O.nzind, O.nzval, 1, length(O.nzind), O.n, mod(r, X.n)) + return O +end + + +circshift!(O::SparseVector, X::SparseVector, r::Real,) = circshift!(O, X, (Integer(r),)) diff --git a/stdlib/SparseArrays/test/sparse.jl b/stdlib/SparseArrays/test/sparse.jl index 163f3b9271f1b..d6e95f1ddb9a9 100644 --- a/stdlib/SparseArrays/test/sparse.jl +++ b/stdlib/SparseArrays/test/sparse.jl @@ -2353,4 +2353,31 @@ end @test success(pipeline(cmd; stdout=stdout, stderr=stderr)) end +@testset "circshift" begin + m,n = 17,15 + A = sprand(m, n, 0.5) + for rshift in (-1, 0, 1, 10), cshift in (-1, 0, 1, 10) + shifts = (rshift, cshift) + # using dense circshift to compare + B = circshift(Matrix(A), shifts) + # sparse circshift + C = circshift(A, shifts) + @test C == B + # sparse circshift should not add structural zeros + @test nnz(C) == nnz(A) + # test circshift! + D = similar(A) + circshift!(D, A, shifts) + @test D == B + @test nnz(D) == nnz(A) + # test different in/out types + A2 = floor.(100A) + E1 = spzeros(Int64, m, n) + E2 = spzeros(Int64, m, n) + circshift!(E1, A2, shifts) + circshift!(E2, Matrix(A2), shifts) + @test E1 == E2 + end +end + end # module diff --git a/stdlib/SparseArrays/test/sparsevector.jl b/stdlib/SparseArrays/test/sparsevector.jl index 6061b21a6721c..8d0f7b7dc1b7e 100644 --- a/stdlib/SparseArrays/test/sparsevector.jl +++ b/stdlib/SparseArrays/test/sparsevector.jl @@ -1260,4 +1260,26 @@ end end end +@testset "SparseVector circshift" begin + n = 100 + v = sprand(n, 0.5) + for shift in (0,-1,1,5,-7,n+10) + x = circshift(Vector(v), shift) + w = circshift(v, shift) + @test nnz(v) == nnz(w) + @test w == x + # test circshift! + v1 = similar(v) + circshift!(v1, v, shift) + @test v1 == x + # test different in/out types + y1 = spzeros(Int64, n) + y2 = spzeros(Int64, n) + v2 = floor.(100v) + circshift!(y1, v2, shift) + circshift!(y2, Vector(v2), shift) + @test y1 == y2 + end +end + end # module From 543cf244caa3baa8fa0924d2067b83d119dc1d11 Mon Sep 17 00:00:00 2001 From: denizyuret Date: Wed, 9 Jan 2019 18:30:45 -0500 Subject: [PATCH 35/41] fix #30643, correctly propagate iterator traits through Stateful (#30644) (cherry picked from commit 21dfef3f1656d952540f394145c24552cd37b0a2) --- base/iterators.jl | 5 ++--- test/iterators.jl | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/base/iterators.jl b/base/iterators.jl index 449e8dbb0663d..118cb9b61c5de 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -1094,10 +1094,9 @@ end @inline peek(s::Stateful, sentinel=nothing) = s.nextvalstate !== nothing ? s.nextvalstate[1] : sentinel @inline iterate(s::Stateful, state=nothing) = s.nextvalstate === nothing ? nothing : (popfirst!(s), nothing) -IteratorSize(::Type{Stateful{VS,T}} where VS) where {T} = - isa(IteratorSize(T), SizeUnknown) ? SizeUnknown() : HasLength() +IteratorSize(::Type{Stateful{T,VS}}) where {T,VS} = IteratorSize(T) isa HasShape ? HasLength() : IteratorSize(T) eltype(::Type{Stateful{T, VS}} where VS) where {T} = eltype(T) -IteratorEltype(::Type{Stateful{VS,T}} where VS) where {T} = IteratorEltype(T) +IteratorEltype(::Type{Stateful{T,VS}}) where {T,VS} = IteratorEltype(T) length(s::Stateful) = length(s.itr) - s.taken end diff --git a/test/iterators.jl b/test/iterators.jl index be5ad0b9398e5..5504fb03df381 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -549,3 +549,26 @@ end @test ps isa Iterators.Pairs @test collect(ps) == [1 => :a, 2 => :b] end + +@testset "Stateful fix #30643" begin + @test Base.IteratorSize(1:10) isa Base.HasShape + a = Iterators.Stateful(1:10) + @test Base.IteratorSize(a) isa Base.HasLength + @test length(a) == 10 + @test length(collect(a)) == 10 + @test length(a) == 0 + b = Iterators.Stateful(Iterators.take(1:10,3)) + @test Base.IteratorSize(b) isa Base.HasLength + @test length(b) == 3 + @test length(collect(b)) == 3 + @test length(b) == 0 + c = Iterators.Stateful(Iterators.countfrom(1)) + @test Base.IteratorSize(c) isa Base.IsInfinite + @test length(Iterators.take(c,3)) == 3 + @test length(collect(Iterators.take(c,3))) == 3 + d = Iterators.Stateful(Iterators.filter(isodd,1:10)) + @test Base.IteratorSize(d) isa Base.SizeUnknown + @test length(collect(Iterators.take(d,3))) == 3 + @test length(collect(d)) == 2 + @test length(collect(d)) == 0 +end From ba9d9814a2749c8a19d63290669168b873c8aab9 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 7 Jan 2019 13:30:25 -0500 Subject: [PATCH 36/41] Fix SROA confusing new and old nodes SROA was accidentally treating a pending node as old and thus getting the wrong type when querying the predecessor. As a result it thought one of the paths was unreachable causing undefined data to be introduced on that path (generally the `1.0` that happened to already be in register). Fix #29983 (cherry picked from commit da0179c4d60c0fa3e5d64c08972773c82d57e4e6) --- base/compiler/ssair/passes.jl | 12 +++++++++--- test/compiler/irpasses.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 test/compiler/irpasses.jl diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index ce04b81bac1eb..83e4fa09968d3 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -191,7 +191,7 @@ function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospe collect(Iterators.filter(1:length(def.edges)) do n isassigned(def.values, n) || return false val = def.values[n] - if isa(defssa, OldSSAValue) && isa(val, SSAValue) + if is_old(compact, defssa) && isa(val, SSAValue) val = OldSSAValue(val.id) end edge_typ = widenconst(compact_exprtype(compact, val)) @@ -201,7 +201,7 @@ function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospe for n in possible_predecessors pred = def.edges[n] val = def.values[n] - if isa(defssa, OldSSAValue) && isa(val, SSAValue) + if is_old(compact, defssa) && isa(val, SSAValue) val = OldSSAValue(val.id) end if isa(val, AnySSAValue) @@ -425,6 +425,12 @@ struct LiftedPhi need_argupdate::Bool end +function is_old(compact, @nospecialize(old_node_ssa)) + isa(old_node_ssa, OldSSAValue) && + !is_pending(compact, old_node_ssa) && + !already_inserted(compact, old_node_ssa) +end + function perform_lifting!(compact::IncrementalCompact, visited_phinodes::Vector{Any}, @nospecialize(cache_key), lifting_cache::IdDict{Pair{AnySSAValue, Any}, AnySSAValue}, @@ -455,7 +461,7 @@ function perform_lifting!(compact::IncrementalCompact, isassigned(old_node.values, i) || continue val = old_node.values[i] orig_val = val - if isa(old_node_ssa, OldSSAValue) && !is_pending(compact, old_node_ssa) && !already_inserted(compact, old_node_ssa) && isa(val, SSAValue) + if is_old(compact, old_node_ssa) && isa(val, SSAValue) val = OldSSAValue(val.id) end if isa(val, Union{NewSSAValue, SSAValue, OldSSAValue}) diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl new file mode 100644 index 0000000000000..e37b9fa5c4a0e --- /dev/null +++ b/test/compiler/irpasses.jl @@ -0,0 +1,29 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test + +# Issue #29983 +# This one is a bit hard to trigger, but the key is to create a case +# where SROA needs to introduce an intermediate type-unstable phi node +struct Foo29983{T} + x::Tuple{T} +end +struct Bar29983{S} + x::S +end +Base.:+(a::T, b::Bar29983{S}) where {T, S} = Bar29983(a + b.x) +Base.:+(a::Bar29983{S}, b::T) where {T, S} = b + a +Base.:+(a::Bar29983{S}, b::Bar29983{T}) where {T, S} = Bar29983(a.x + b.x) +Base.:+(a::Foo29983, b::Foo29983) = Foo29983((a.x[1] + b.x[1],)) + +function f(x::Vector{T}) where {T} + x1 = Foo29983((x[1],)) + la1 = Foo29983((x[1],)) + f1 = Foo29983((0,)) + for _ in 1:2 + f1 += la1 + end + return f1 +end + +@test f([Bar29983(1.0)]).x[1].x == 2.0 From 28ca40c352de6baae3dd0c57d4b073453c19ea76 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 4 Jan 2019 22:08:17 -0500 Subject: [PATCH 37/41] Defensively fix patterns similar to #29983 I don't have concrete tests for these, but it looks like they all need the `is_old` predicate for what they're doing, so switch those over also while we're at it. (cherry picked from commit 34f7a4a50458594e6d72793b4f371227e8e842e3) --- base/compiler/ssair/passes.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 83e4fa09968d3..887da342b663c 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -130,7 +130,7 @@ function simple_walk(compact::IncrementalCompact, @nospecialize(defssa#=::AnySSA return defssa end if isa(def.val, SSAValue) - if isa(defssa, OldSSAValue) && !already_inserted(compact, defssa) + if is_old(compact, defssa) defssa = OldSSAValue(def.val.id) else defssa = def.val @@ -281,7 +281,7 @@ function lift_leaves(compact::IncrementalCompact, @nospecialize(stmt), end if is_tuple_call(compact.ir, def) && isa(field, Int) && 1 <= field < length(def.args) lifted = def.args[1+field] - if isa(leaf, OldSSAValue) && isa(lifted, SSAValue) + if is_old(compact, leaf) && isa(lifted, SSAValue) lifted = OldSSAValue(lifted.id) end if isa(lifted, GlobalRef) || isa(lifted, Expr) @@ -320,7 +320,7 @@ function lift_leaves(compact::IncrementalCompact, @nospecialize(stmt), compact[leaf] = def end lifted = def.args[1+field] - if isa(leaf, OldSSAValue) && isa(lifted, SSAValue) + if is_old(compact, leaf) && isa(lifted, SSAValue) lifted = OldSSAValue(lifted.id) end if isa(lifted, GlobalRef) || isa(lifted, Expr) @@ -339,7 +339,7 @@ function lift_leaves(compact::IncrementalCompact, @nospecialize(stmt), # N.B.: This can be a bit dangerous because it can lead to # infinite loops if we accidentally insert a node just ahead # of where we are - if isa(leaf, OldSSAValue) && (isa(field, Int) || isa(field, Symbol)) + if is_old(compact, leaf) && (isa(field, Int) || isa(field, Symbol)) (isa(typ, DataType) && (!typ.abstract)) || return nothing @assert !typ.mutable # If there's the potential for an undefref error on access, we cannot insert a getfield From cb77cc335db739c1ec394d5e8f8e9a1f4a43d24b Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Wed, 28 Feb 2018 12:37:14 -0500 Subject: [PATCH 38/41] string doc clarifications Clarify that `firstindex(str)` should always be `1` for any `AbstractString`, as mentioned by @StefanKarpinski [here](https://github.com/JuliaLang/julia/pull/26133#discussion_r170899348). Also reference `prevind` and `eachindex`. Also introduce the "code unit" terminology and mention the `codeunit` functions. (cherry picked from commit 3b6773dae05ee8a93cb2a494895dc300a2147b57) --- doc/src/manual/strings.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index 39442c70b948f..c5bf3a06bb7da 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -180,12 +180,12 @@ julia> str[end] ``` Many Julia objects, including strings, can be indexed with integers. The index of the first -element is returned by [`firstindex(str)`](@ref), and the index of the last element +element (the first character of a string) is returned by [`firstindex(str)`](@ref), and the index of the last element (character) with [`lastindex(str)`](@ref). The keyword `end` can be used inside an indexing operation as shorthand for the last index along the given dimension. -Most indexing in Julia is 1-based: the first element of many integer-indexed objects is found at -index 1. (As we will see below, this does not necessarily mean that the last element is found -at index `n`, where `n` is the length of the string.) +String indexing, like most indexing in Julia, is 1-based: `firstindex` always returns `1` for any `AbstractString`. +As we will see below, however, `lastindex(str)` is *not* in general the same as `length(str)` for a string, +because some Unicode characters can occupy multiple "code units". You can perform arithmetic and other operations with [`end`](@ref), just like a normal value: @@ -265,10 +265,13 @@ julia> s = "\u2200 x \u2203 y" Whether these Unicode characters are displayed as escapes or shown as special characters depends on your terminal's locale settings and its support for Unicode. String literals are encoded using the UTF-8 encoding. UTF-8 is a variable-width encoding, meaning that not all characters are encoded -in the same number of bytes. In UTF-8, ASCII characters -- i.e. those with code points less than +in the same number of bytes ("code units"). In UTF-8, ASCII characters — i.e. those with code points less than 0x80 (128) -- are encoded as they are in ASCII, using a single byte, while code points 0x80 and -above are encoded using multiple bytes -- up to four per character. This means that not every -byte index into a UTF-8 string is necessarily a valid index for a character. If you index into +above are encoded using multiple bytes — up to four per character. + +String indices in Julia refer to code units (= bytes for UTF-8), the fixed-width building blocks that +are used to encode arbitrary characters (code points). This means that not every +index into a `String` is necessarily a valid index for a character. If you index into a string at such an invalid byte index, an error is thrown: ```jldoctest unicodestring @@ -348,6 +351,26 @@ x y ``` +If you need to obtain valid indices for a string, you can use the [`nextind`](@ref) and +[`prevind`](@ref) functions to increment/decrement to the next/previous valid index, as mentioned above. +You can also use the [`eachindex`](@ref) function to iterate over the valid character indices: + +```jldoctest unicodestring +julia> collect(eachindex(s)) +7-element Array{Int64,1}: + 1 + 4 + 5 + 6 + 7 + 10 + 11 +``` + +To access the raw code units (bytes for UTF-8) of the encoding, you can use the [`codeunit(s,i)`](@ref) +function, where the index `i` runs consecutively from `1` to [`ncodeunits(s)`](@ref). The [`codeunits(s)`](@ref) +function returns an `AbstractVector{UInt8}` wrapper that lets you access these raw codeunits (bytes) as an array. + Strings in Julia can contain invalid UTF-8 code unit sequences. This convention allows to treat any byte sequence as a `String`. In such situations a rule is that when parsing a sequence of code units from left to right characters are formed by the longest sequence of From bac93503b16bfaa6b2de82f9b281f4391c6ac211 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Thu, 10 Jan 2019 11:53:14 -0500 Subject: [PATCH 39/41] string docs: clarify handling of overlong and too-high sequences (cherry picked from commit 8b45c9c4a97c0492b14b8a3692a66eff99ba58e7) --- doc/src/manual/strings.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index c5bf3a06bb7da..d689ed73ec555 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -384,8 +384,9 @@ a sequence of code units from left to right characters are formed by the longest * `10xxxxxx`; * `11111xxx`. -In particular this implies that overlong and too high code unit sequences are accepted. -This rule is best explained by an example: +In particular this means that overlong and too-high code unit sequences and prefixes thereof are treated +as a single invalid character rather than multiple invalid characters. +This rule may be best explained with an example: ```julia-repl julia> s = "\xc0\xa0\xe2\x88\xe2|" From da5d637adc93d98ba7078165e97ad545b030abef Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 4 Jan 2019 19:35:28 -0500 Subject: [PATCH 40/41] Fix use counts for mutable struct SROA PR #28478 moved the computation of the use counts before the finish call. to fix #28444. However, the early parts of the finish call fixes up phi node arguments, which fail to get counted if we look at use counts before that fixup is performed. This causes #30594 where the only non-trivial use is on the backedge of the phi and would thus incorrectly fail to get accounted for. Fix that by taking the use count after phi fixup but before dce. (cherry picked from commit f8f20453c4ca7a6aeb23a189a420dbff130679ce) --- base/compiler/ssair/passes.jl | 10 +++++++--- test/compiler/irpasses.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 887da342b663c..12cd0cd582f8c 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -694,10 +694,14 @@ function getfield_elim_pass!(ir::IRCode, domtree::DomTree) compact[idx] = val === nothing ? nothing : val.x end - # Copy the use count, `finish` may modify it and for our predicate - # below we need it consistent with the state of the IR here. + + non_dce_finish!(compact) + # Copy the use count, `simple_dce!` may modify it and for our predicate + # below we need it consistent with the state of the IR here (after tracking + # phi node arguments, but before dce). used_ssas = copy(compact.used_ssas) - ir = finish(compact) + simple_dce!(compact) + ir = complete(compact) # Now go through any mutable structs and see which ones we can eliminate for (idx, (intermediaries, defuse)) in defuses intermediaries = collect(intermediaries) diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl index e37b9fa5c4a0e..91bb336a5ab9a 100644 --- a/test/compiler/irpasses.jl +++ b/test/compiler/irpasses.jl @@ -2,6 +2,32 @@ using Test +# Tests for SROA + +mutable struct Foo30594; x::Float64; end +Base.copy(x::Foo30594) = Foo30594(x.x) +function add!(p::Foo30594, off::Foo30594) + p.x += off.x + return p +end +Base.:(+)(a::Foo30594, b::Foo30594) = add!(copy(a), b) + +let results = Float64[] + @noinline use30594(x) = push!(results, x.x); nothing + function foo30594(cnt::Int, dx::Int) + step = Foo30594(dx) + curr = step + Foo30594(1) + for i in 1:cnt + use30594(curr) + curr = curr + step + end + nothing + end + + foo30594(4, -1) + @test results == [0.0, -1.0, -2.0, -3.0] +end + # Issue #29983 # This one is a bit hard to trigger, but the key is to create a case # where SROA needs to introduce an intermediate type-unstable phi node From f487cf33d519b1debb07bd6fff553e1a3be5e6db Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 3 Jan 2019 13:12:50 -0500 Subject: [PATCH 41/41] llvm: fix target triple (#30554) broken by their move to cmake causing a switch away from the standard --host/--build autoconf fix #28046 (cherry picked from commit 041c21498df49d39d0447372705f9dec4bdd17e7) --- deps/llvm.mk | 2 +- src/disasm.cpp | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 5af5f6363d5c6..d09579b21e27e 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -65,7 +65,7 @@ LLVM_CXXFLAGS += $(CXXFLAGS) LLVM_CPPFLAGS += $(CPPFLAGS) LLVM_LDFLAGS += $(LDFLAGS) LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING="$(LLVM_TARGETS)" -DCMAKE_BUILD_TYPE="$(LLVM_CMAKE_BUILDTYPE)" -LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_LIBXML2=OFF +LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))" ifeq ($(USE_POLLY_ACC),1) LLVM_CMAKE += -DPOLLY_ENABLE_GPGPU_CODEGEN=ON endif diff --git a/src/disasm.cpp b/src/disasm.cpp index c7f74765cba65..bff8dd91f697a 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -628,24 +628,23 @@ static void jl_dump_asm_internal( { // GC safe // Get the host information - std::string TripleName = sys::getDefaultTargetTriple(); - Triple TheTriple(Triple::normalize(TripleName)); + Triple TheTriple(sys::getProcessTriple()); const auto &target = jl_get_llvm_disasm_target(); const auto &cpu = target.first; const auto &features = target.second; std::string err; - const Target *TheTarget = TargetRegistry::lookupTarget(TripleName, err); + const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple.str(), err); // Set up required helpers and streamer std::unique_ptr Streamer; SourceMgr SrcMgr; - std::unique_ptr MAI(TheTarget->createMCAsmInfo(*TheTarget->createMCRegInfo(TripleName),TripleName)); + std::unique_ptr MAI(TheTarget->createMCAsmInfo(*TheTarget->createMCRegInfo(TheTriple.str()), TheTriple.str())); assert(MAI && "Unable to create target asm info!"); - std::unique_ptr MRI(TheTarget->createMCRegInfo(TripleName)); + std::unique_ptr MRI(TheTarget->createMCRegInfo(TheTriple.str())); assert(MRI && "Unable to create target register info!"); std::unique_ptr MOFI(new MCObjectFileInfo()); @@ -659,16 +658,15 @@ static void jl_dump_asm_internal( // Set up Subtarget and Disassembler std::unique_ptr - STI(TheTarget->createMCSubtargetInfo(TripleName, cpu, features)); + STI(TheTarget->createMCSubtargetInfo(TheTriple.str(), cpu, features)); std::unique_ptr DisAsm(TheTarget->createMCDisassembler(*STI, Ctx)); if (!DisAsm) { - jl_printf(JL_STDERR, "ERROR: no disassembler for target %s\n", - TripleName.c_str()); + rstream << "ERROR: no disassembler for target " << TheTriple.str(); return; } unsigned OutputAsmVariant = 0; // ATT or Intel-style assembly - if (strcmp(asm_variant, "intel")==0) { + if (strcmp(asm_variant, "intel") == 0) { OutputAsmVariant = 1; } bool ShowEncoding = false;