From a6f9c6a5297e1ba8adddbed182738966ec075e40 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 16 Jun 2018 18:46:50 -0700 Subject: [PATCH 01/12] Properly mark a deleted typemap entry (#27568) This issue possibly fixes #24951 (or at least the test case by iamed2). We believe the original code here meant to say either: ((jl_typemap_entry_t*)v)->min_world = ((jl_typemap_entry_t*)v)->max_world + 1; or ((jl_typemap_entry_t*)v)->max_world = ((jl_typemap_entry_t*)v)->min_world - 1; i.e. set the range of applicable worlds to be empty. What happened instead was that the given typemap entry that was supposed to be deleted became valid for one particular world and that world only. Thus any code running in that particular world may try to access the deleted typemap entry (or add a backedge to it), causing either incorrect behavior or the assertion failure noted in the issue. One additional complication is that these world ages are being deserialized, i.e. they may be larger than the currently possible max world age. This makes this slightly more likely to happen, since the current process may work its way up to that world age and exectue some code. In any case, there's not much value to keeping around the deserialized max or min world, so just mark them as [1:0], as we do for other deleted entries. Co-authored-by: Jameson Nash --- NOTE: This backported commit EXCLUDES additional assertions made by vtjnash. (Cherry-picked from commit d9b10f08fa0bd65cb49c02661e2784ffda9061f3) --- src/dump.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dump.c b/src/dump.c index c49bc2168e79f..68d5a62713695 100644 --- a/src/dump.c +++ b/src/dump.c @@ -2003,7 +2003,8 @@ static void jl_deserialize_struct(jl_serializer_state *s, jl_value_t *v, size_t } else { // garbage entry - delete it :( - ((jl_typemap_entry_t*)v)->min_world = ((jl_typemap_entry_t*)v)->max_world - 1; + ((jl_typemap_entry_t*)v)->min_world = 1; + ((jl_typemap_entry_t*)v)->max_world = 0; } } } From 79b32587ca9af7d3f8894e42d9ce8f05603d3fb4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 4 Jun 2018 22:19:53 -0400 Subject: [PATCH 02/12] fix #27361, error adding method using e.g. `Tuple{3}` (#27404) (cherry picked from commit f89921fb36c34addee47b1ab93cb49d9d7b3d934) --- src/subtype.c | 6 ++++-- test/specificity.jl | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index b71aa76609d56..5799ef59aa346 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2227,8 +2227,10 @@ static int sub_msp(jl_value_t *a, jl_value_t *b, jl_typeenv_t *env) { JL_GC_PUSH2(&a, &b); while (env != NULL) { - a = jl_type_unionall(env->var, a); - b = jl_type_unionall(env->var, b); + if (jl_is_type(a) || jl_is_typevar(a)) + a = jl_type_unionall(env->var, a); + if (jl_is_type(b) || jl_is_typevar(b)) + b = jl_type_unionall(env->var, b); env = env->prev; } int sub = jl_subtype(a, b); diff --git a/test/specificity.jl b/test/specificity.jl index 0273a93b2d582..491b1c3c27660 100644 --- a/test/specificity.jl +++ b/test/specificity.jl @@ -194,3 +194,8 @@ let A = Tuple{Vector, AbstractVector}, @test args_morespecific(B, C) @test args_morespecific(A, C) end + +# issue #27361 +f27361(::M) where M <: Tuple{2} = nothing +f27361(::M) where M <: Tuple{3} = nothing +@test length(methods(f27361)) == 2 From 93b0f7c1f7bf654d2cb8965d043c890ab8ff7c63 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Sat, 16 Jun 2018 20:35:57 -0700 Subject: [PATCH 03/12] Fix naivesub Bidigonal to work for matrix elements (#27203) by avoiding calling zero(T). Also a few minor performance improvements. (Cherry-picked from commit 3fa83f94f161829f46f24045c718d037a99393ce) --- base/linalg/bidiag.jl | 43 +++++++++++++++++++++++++++++++------------ test/linalg/bidiag.jl | 12 ++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 9e9af0bb890ff..90d62f0cc1c7c 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -543,20 +543,39 @@ function naivesub!(A::Bidiagonal{T}, b::AbstractVector, x::AbstractVector = b) w if N != length(b) || N != length(x) throw(DimensionMismatch("second dimension of A, $N, does not match one of the lengths of x, $(length(x)), or b, $(length(b))")) end - if !A.isupper #do forward substitution - for j = 1:N - x[j] = b[j] - j > 1 && (x[j] -= A.ev[j-1] * x[j-1]) - x[j] /= A.dv[j] == zero(T) ? throw(SingularException(j)) : A.dv[j] - end - else #do backward substitution - for j = N:-1:1 - x[j] = b[j] - j < N && (x[j] -= A.ev[j] * x[j+1]) - x[j] /= A.dv[j] == zero(T) ? throw(SingularException(j)) : A.dv[j] + + if N == 0 + return x + end + + @inbounds begin + if !A.isupper #do forward substitution + x[1] = xj1 = A.dv[1]\b[1] + for j = 2:N + xj = b[j] + xj -= A.ev[j - 1] * xj1 + dvj = A.dv[j] + if iszero(dvj) + throw(SingularException(j)) + end + xj = dvj\xj + x[j] = xj1 = xj + end + else #do backward substitution + x[N] = xj1 = A.dv[N]\b[N] + for j = (N - 1):-1:1 + xj = b[j] + xj -= A.ev[j] * xj1 + dvj = A.dv[j] + if iszero(dvj) + throw(SingularException(j)) + end + xj = dvj\xj + x[j] = xj1 = xj + end end end - x + return x end ### Generic promotion methods and fallbacks diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index 7e128844eec7a..2c3a892a1e892 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -328,3 +328,15 @@ import Base.LinAlg: fillslots!, UnitLowerTriangular end end end + +@testset "solve with matrix elements" begin + A = triu(tril(randn(9, 9), 3), -3) + b = randn(9) + Alb = Bidiagonal(Any[tril(A[1:3,1:3]), tril(A[4:6,4:6]), tril(A[7:9,7:9])], + Any[triu(A[4:6,1:3]), triu(A[7:9,4:6])], 'L') + Aub = Bidiagonal(Any[triu(A[1:3,1:3]), triu(A[4:6,4:6]), triu(A[7:9,7:9])], + Any[tril(A[1:3,4:6]), tril(A[4:6,7:9])], 'U') + bb = Any[b[1:3], b[4:6], b[7:9]] + @test vcat((Alb\bb)...) ≈ LowerTriangular(A)\b + @test vcat((Aub\bb)...) ≈ UpperTriangular(A)\b +end From 049cb5932451cd67da183544f68b51112bfbb16b Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Sat, 2 Jun 2018 17:33:40 -0700 Subject: [PATCH 04/12] Remove build directories from RPATH on FreeBSD Currently we're putting build directories as well as GCC's directory directly into our libraries' RPATHs. This is fine for simple source builds that sit where they were built and aren't distributed, like on CI. However, this causes major issues for distributing. Ref #27396 (cherry picked from commit 5a426b07c35b5956d77cb102a8d270b3cbe8cab3) --- Make.inc | 8 ++++---- Makefile | 18 ++++++++++++++++++ contrib/fixup-rpath.sh | 30 ++++++++++++++++++++++++++++++ deps/blas.mk | 1 + 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100755 contrib/fixup-rpath.sh diff --git a/Make.inc b/Make.inc index 7f3a37c01d056..f8aa57e91da02 100644 --- a/Make.inc +++ b/Make.inc @@ -539,10 +539,10 @@ endif # Allow the user to specify this in Make.user GCCPATH ?= $(LOCALBASE)/lib/gcc$(_GCCVER) -LDFLAGS += -L$(build_libdir) -L$(GCCPATH) -Wl,-rpath,$(build_libdir) -Wl,-rpath,$(GCCPATH) - -# This ensures we get the right RPATH even if we're missing FFLAGS somewhere -FC += -Wl,-rpath=$(GCCPATH) +# We're going to copy over the libraries we need from GCCPATH into build_libdir, then +# tell everyone to look for them there. At install time, the build_libdir added into +# the RPATH here is removed by patchelf. +LDFLAGS += -L$(build_libdir) -Wl,-rpath,$(build_libdir) endif # gfortran endif # FreeBSD diff --git a/Makefile b/Makefile index f7417e1572089..29b5376562179 100644 --- a/Makefile +++ b/Makefile @@ -309,6 +309,19 @@ endif endif endif +ifeq ($(OS),FreeBSD) +define std_so +julia-deps: | $$(build_libdir)/lib$(1).so +$$(build_libdir)/lib$(1).so: | $$(build_libdir) + $$(INSTALL_M) $$(GCCPATH)/lib$(1).so* $$(build_libdir) +JL_LIBS += $(1) +endef + +$(eval $(call std_so,gfortran)) +$(eval $(call std_so,gcc_s)) +$(eval $(call std_so,quadmath)) +endif # FreeBSD + ifeq ($(OS),WINNT) define std_dll julia-deps: | $$(build_bindir)/lib$(1).dll $$(build_depsbindir)/lib$(1).dll @@ -417,6 +430,11 @@ endif $(call stringreplace,$(DESTDIR)$(libdir)/libjulia-debug.$(SHLIB_EXT),sys-debug.$(SHLIB_EXT)$$,$(private_libdir_rel)/sys-debug.$(SHLIB_EXT)) endif + # On FreeBSD, remove the build's libdir from each library's RPATH +ifeq ($(OS),FreeBSD) + $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(libdir) $(build_libdir) +endif + mkdir -p $(DESTDIR)$(sysconfdir) cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/ diff --git a/contrib/fixup-rpath.sh b/contrib/fixup-rpath.sh new file mode 100755 index 0000000000000..342cf42e9a73c --- /dev/null +++ b/contrib/fixup-rpath.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Usage: fixup-rpath.sh + +if [ $# -ne 3 ]; then + echo "Incorrect number of arguments: Expected 3, got $#" + echo "Usage: fixup-rpath.sh " + exit 1 +fi + +patchelf="$1" +install_libdir="$2" +build_libdir="$3" + +for lib in ${install_libdir}/*.so*; do + # First get the current RPATH + rpath="$(${patchelf} --print-rpath ${lib})" + + # If it doesn't contain the build's libdir, we don't care about it + if [ -z "$(echo ${rpath} | grep -F ${build_libdir})" ]; then + continue + fi + + # Remove build_libdir from the RPATH, retaining the rest + new_rpath="$(echo ${rpath} | tr : \\n | grep -vF ${build_libdir} | tr \\n :)" + # Drop the trailing : + new_rpath="${new_rpath%?}" + + # Now set the new RPATH + ${patchelf} --set-rpath "${new_rpath}" ${lib} +done diff --git a/deps/blas.mk b/deps/blas.mk index e81c96be99fab..c6730a14d66b5 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -64,6 +64,7 @@ endif OPENBLAS_BUILD_OPTS += CFLAGS="$(CFLAGS) $(OPENBLAS_CFLAGS)" OPENBLAS_BUILD_OPTS += FFLAGS="$(FFLAGS) $(OPENBLAS_FFLAGS)" +OPENBLAS_BUILD_OPTS += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" # Debug OpenBLAS ifeq ($(OPENBLAS_DEBUG), 1) From 96c075a41991e9754e4ab5983afd95df17e32dde Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 18 Jun 2018 10:12:18 -0700 Subject: [PATCH 05/12] Fix executable and more library RPATHs on FreeBSD (#27627) First of all, we need to add libgfortran, libgcc_s, and libquadmath to `JL_PRIVATE_LIBS-0` instead of `JL_LIBS`. This ensures that they get installed into `lib/julia`, where our dependencies are supposed to live, instead of `lib`, where only libjulia should live. Now we need to call `fixup-rpath.sh` on more directories. Currently we're only calling it on `lib`, but we also need to call it on `lib/julia` and even on `bin`, because FreeBSD seems to load libraries based on the executable's RPATH rather than libjulia's RPATH. AND, speaking of the executable's RPATH, we need to make sure we have `lib/julia` in there; currently we only have `lib`. So in an isolated environment outside of the build directory, Julia won't be able to load any of its dependencies, which is bad. This is accomplished with a simple adjustment in `Make.inc`. To make sure that our dependencies can find each other properly, we need to add `RPATH_ESCAPED_ORIGIN` to `LDFLAGS` in `CONFIGURE_COMMON`. That way, every dependency that uses `configure` will get its RPATH set appropriately. (cherry picked from commit 07edeadc326a4af1ed867f8b8cf649f18fc01df9) --- Make.inc | 3 +++ Makefile | 34 ++++++++++++++++++++++++++-------- contrib/fixup-rpath.sh | 10 ++++++---- deps/tools/common.mk | 2 ++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Make.inc b/Make.inc index f8aa57e91da02..ac8d8a1b71ac1 100644 --- a/Make.inc +++ b/Make.inc @@ -934,6 +934,9 @@ else ifeq ($(OS), Darwin) RPATH_LIB := -Wl,-rpath,'@loader_path/julia/' -Wl,-rpath,'@loader_path/' else RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin +ifeq ($(OS), FreeBSD) + RPATH += -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' +endif RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin RPATH_LIB := -Wl,-rpath,'$$ORIGIN/julia' -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin diff --git a/Makefile b/Makefile index 29b5376562179..cf8c9e05567ae 100644 --- a/Makefile +++ b/Makefile @@ -309,17 +309,22 @@ endif endif endif +# On FreeBSD, /lib/libgcc_s.so.1 is incompatible with Fortran; to use Fortran on FreeBSD, +# we need to link to the libgcc_s that ships with the same GCC version used by libgfortran. +# To work around this, we copy the GCC libraries we need, namely libgfortran, libgcc_s, +# and libquadmath, into our build library directory, $(build_libdir). We also add them to +# JL_PRIVATE_LIBS so that they know where they need to live at install time. ifeq ($(OS),FreeBSD) define std_so -julia-deps: | $$(build_libdir)/lib$(1).so -$$(build_libdir)/lib$(1).so: | $$(build_libdir) - $$(INSTALL_M) $$(GCCPATH)/lib$(1).so* $$(build_libdir) -JL_LIBS += $(1) +julia-deps: | $$(build_libdir)/$(1).so +$$(build_libdir)/$(1).so: | $$(build_libdir) + $$(INSTALL_M) $$(GCCPATH)/$(1).so* $$(build_libdir) +JL_PRIVATE_LIBS += $(1) endef -$(eval $(call std_so,gfortran)) -$(eval $(call std_so,gcc_s)) -$(eval $(call std_so,quadmath)) +$(eval $(call std_so,libgfortran)) +$(eval $(call std_so,libgcc_s)) +$(eval $(call std_so,libquadmath)) endif # FreeBSD ifeq ($(OS),WINNT) @@ -433,6 +438,15 @@ endif # On FreeBSD, remove the build's libdir from each library's RPATH ifeq ($(OS),FreeBSD) $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(libdir) $(build_libdir) + $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(private_libdir) $(build_libdir) + $(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(bindir) $(build_libdir) + # Set libgfortran's RPATH to ORIGIN instead of GCCPATH. It's only libgfortran that + # needs to be fixed here, as libgcc_s and libquadmath don't have RPATHs set. If we + # don't set libgfortran's RPATH, it won't be able to find its friends on systems + # that don't have the exact GCC port installed used for the build. + for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \ + $(build_depsbindir)/patchelf --set-rpath '$$ORIGIN' $$lib; \ + done endif mkdir -p $(DESTDIR)$(sysconfdir) @@ -461,7 +475,11 @@ ifneq ($(DESTDIR),) endif @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT) -ifneq ($(OS), WINNT) + # Run fixup-libgfortran on all platforms but Windows and FreeBSD. On FreeBSD we + # pull in the GCC libraries earlier and use them for the build to make sure we + # don't inadvertently link to /lib/libgcc_s.so.1, which is incompatible with + # libgfortran. +ifeq (,$(findstring $(OS),FreeBSD WINNT)) -$(CUSTOM_LD_LIBRARY_PATH) PATH=$(PATH):$(build_depsbindir) $(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) endif ifeq ($(OS), Linux) diff --git a/contrib/fixup-rpath.sh b/contrib/fixup-rpath.sh index 342cf42e9a73c..10e0790724667 100755 --- a/contrib/fixup-rpath.sh +++ b/contrib/fixup-rpath.sh @@ -1,17 +1,17 @@ #!/bin/sh -# Usage: fixup-rpath.sh +# Usage: fixup-rpath.sh if [ $# -ne 3 ]; then echo "Incorrect number of arguments: Expected 3, got $#" - echo "Usage: fixup-rpath.sh " + echo "Usage: fixup-rpath.sh " exit 1 fi patchelf="$1" -install_libdir="$2" +executable_dir="$2" build_libdir="$3" -for lib in ${install_libdir}/*.so*; do +for lib in $(find ${executable_dir} -type f -perm -111); do # First get the current RPATH rpath="$(${patchelf} --print-rpath ${lib})" @@ -25,6 +25,8 @@ for lib in ${install_libdir}/*.so*; do # Drop the trailing : new_rpath="${new_rpath%?}" + echo " Setting RPATH for ${lib} to '${new_rpath}'" + # Now set the new RPATH ${patchelf} --set-rpath "${new_rpath}" ${lib} done diff --git a/deps/tools/common.mk b/deps/tools/common.mk index e595ff9cc0c4b..bc721de108782 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -12,6 +12,8 @@ ifeq ($(OS),WINNT) ifneq ($(USEMSVC), 1) CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608" endif +else +CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" endif CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXXFLAGS)" From 161216ae862a299374ddaeed97bff326b8c20096 Mon Sep 17 00:00:00 2001 From: ronisbr Date: Mon, 18 Jun 2018 16:30:28 -0700 Subject: [PATCH 06/12] Fix test with libgit2 v0.26.0+ This fixes an error in tests when libgit2 version is equal or newer v0.26.0. Closes #24453 Ref #27406 --- test/libgit2.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index 0f09a6ff02d84..d643f6b49bb75 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -1697,7 +1697,7 @@ mktempdir() do dir deserialize(f) end @test err.code == LibGit2.Error.ERROR - @test err.msg == "Invalid Content-Type: text/plain" + @test lowercase(err.msg) == lowercase("Invalid Content-Type: text/plain") end finally kill(pobj) From df1c1c939bf1c582cf040b5ad235933242aeacfb Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 18 Jun 2018 17:38:37 -0400 Subject: [PATCH 07/12] fix #27632, bug in subtyping length-constrained varargs Ref #27646 (cherry picked from commit 8e3120c41d7e03badc3c2d610daf9541721e1a92) --- src/subtype.c | 15 +++++++++++++++ test/subarray.jl | 10 ++++++++++ test/subtype.jl | 7 +++++++ 3 files changed, 32 insertions(+) diff --git a/src/subtype.c b/src/subtype.c index 5799ef59aa346..00049bc906ea6 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -737,6 +737,21 @@ static int subtype_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_stenv_t *e, in break; // if y ends in `Vararg{Any}` skip checking everything } if (vx && vy) { + jl_tvar_t *yp1=NULL, *yp2=NULL; + jl_value_t *yva = unwrap_2_unionall(yi, &yp1, &yp2); + jl_tvar_t *xp1=NULL, *xp2=NULL; + jl_value_t *xva = unwrap_2_unionall(xi, &xp1, &xp2); + if ((jl_value_t*)xp1 == jl_tparam1(xva) || (jl_value_t*)xp2 == jl_tparam1(xva)) { + // check for unconstrained vararg on left, constrained on right + if (jl_is_long(jl_tparam1(yva))) + return 0; + if (jl_is_typevar(jl_tparam1(yva))) { + jl_varbinding_t *ylv = lookup(e, (jl_tvar_t*)jl_tparam1(yva)); + if (ylv && jl_is_long(ylv->lb)) + return 0; + } + } + // skip testing element type if vararg lengths are 0 if (jl_is_datatype(xi)) { jl_value_t *xl = jl_tparam1(xi); diff --git a/test/subarray.jl b/test/subarray.jl index 45d31b6fe3ecd..bc610aa1f1f96 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -563,3 +563,13 @@ let su = view(u, :, 1) @test size(@inferred(xcorr(x, su))) == (19,) end + +# issue #27632 +function _test_27632(A) + for J in CartesianRange(size(A)[2:end]) + A[1, J] + end + nothing +end +# check that this doesn't crash +_test_27632(view(ones(Int64, (1, 1, 1)), 1, 1, 1)) diff --git a/test/subtype.jl b/test/subtype.jl index 2756241a04343..9945f6a433b37 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -1179,3 +1179,10 @@ for it = 1:5 end @test round.(x_24305, 2) == [1.78, 1.42, 1.24] + +# issue #27632 +@test !(Tuple{Array{Int,0}, Int, Vararg{Int}} <: Tuple{AbstractArray{T,N}, Vararg{Int,N}} where {T, N}) +@test !(Tuple{Array{Int,0}, Int, Vararg{Int}} <: Tuple{AbstractArray{T,N}, Vararg{Any,N}} where {T, N}) +@test !(Tuple{Array{Int,0}, Vararg{Any}} <: Tuple{AbstractArray{T,N}, Vararg{Any,N}} where {T, N}) +@test Tuple{Array{Int,0},} <: Tuple{AbstractArray{T,N}, Vararg{Any,N}} where {T, N} +@test !(Tuple{Array{Int,0}, Any} <: Tuple{AbstractArray{T,N}, Vararg{Any,N}} where {T, N}) From 24c36747cf90470f01c915188198e01b9f4a5468 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Tue, 19 Jun 2018 20:59:09 -0700 Subject: [PATCH 08/12] Revert "add AbstractVecOrMat fallback for vcat (fixes #25770) (#25777)" This reverts commit 3a27e01c7793a12bedd1282bbf77263614ffe905. As noted in #27652, this change is breaking. --- base/abstractarray.jl | 6 +++--- test/abstractarray.jl | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 3318bc948b786..8df9186d4ab07 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1129,10 +1129,10 @@ function typed_hcat(::Type{T}, A::AbstractVecOrMat...) where T return B end -vcat(A::AbstractVecOrMat...) = typed_vcat(promote_eltype(A...), A...) -vcat(A::AbstractVecOrMat{T}...) where {T} = typed_vcat(T, A...) +vcat(A::AbstractMatrix...) = typed_vcat(promote_eltype(A...), A...) +vcat(A::AbstractMatrix{T}...) where {T} = typed_vcat(T, A...) -function typed_vcat(::Type{T}, A::AbstractVecOrMat...) where T +function typed_vcat(::Type{T}, A::AbstractMatrix...) where T nargs = length(A) nrows = sum(a->size(a, 1), A)::Int ncols = size(A[1], 2) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index ac9353dd80442..94d301eb58873 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -846,9 +846,3 @@ end Z = Array{Int}(); Z[] = 17 @test Z == collect(Z) == copy(Z) end - -@testset "issue #25770" begin - @test vcat(1:3, fill(1, (2,1))) == vcat([1:3;], fill(1, (2,1))) == reshape([1,2,3,1,1], 5,1) - @test hcat(1:2, fill(1, (2,1))) == hcat([1:2;], fill(1, (2,1))) == reshape([1,2,1,1],2,2) - @test [(1:3) (4:6); fill(1, (3,2))] == reshape([1,2,3,1,1,1,4,5,6,1,1,1], 6,2) -end From 532f31ecbfa1afd3a8b3113c41d6dd28e3677b47 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Fri, 22 Jun 2018 20:56:25 -0700 Subject: [PATCH 09/12] function cache: fix memory leak (#27739) some lookup functions (most notably, jl_get_specialization1) were bypassing the initial cache lookup, resulting in repeatedly trying to re-cache them, and wasting a small amount memory also remove some fast-path code for threading race conditions: the benefit is too negligible to be worth the extra code maintenance (cherry-pick of e36ad5c5151d13a97a51d2ddb5ac1c50555c5595, PR #26982) --- src/gf.c | 104 +++++++++++++++---------------------------- src/julia_internal.h | 2 - 2 files changed, 37 insertions(+), 69 deletions(-) diff --git a/src/gf.c b/src/gf.c index cdeaae00a7e52..86c0bed3e66bc 100644 --- a/src/gf.c +++ b/src/gf.c @@ -829,6 +829,10 @@ static jl_method_instance_t *cache_method(jl_methtable_t *mt, union jl_typemap_t // caller must hold the mt->writelock jl_method_t *definition = m->func.method; jl_value_t *decl = (jl_value_t*)m->sig; + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(*cache, tt, NULL, /*inexact*/0, /*subtype*/1, jl_cachearg_offset(mt), world); + if (entry && entry->func.value) + return (jl_method_instance_t*)entry->func.value; + jl_value_t *temp = NULL; jl_value_t *temp2 = NULL; jl_value_t *temp3 = NULL; @@ -1030,12 +1034,18 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype { // caller must hold the mt->writelock jl_typemap_entry_t *entry = NULL; + entry = jl_typemap_assoc_by_type(mt->cache, tt, NULL, inexact, /*subtype*/1, jl_cachearg_offset(mt), world); + if (entry != NULL && entry != INEXACT_ENTRY && entry->func.value != NULL) { + assert(entry->func.linfo->min_world <= entry->min_world && entry->func.linfo->max_world >= entry->max_world && + "typemap consistency error: MethodInstance doesn't apply to full range of its entry"); + return entry->func.linfo; + } + jl_svec_t *env = jl_emptysvec; - jl_method_t *func = NULL; jl_tupletype_t *sig = NULL; - JL_GC_PUSH4(&env, &entry, &func, &sig); + JL_GC_PUSH2(&env, &sig); - entry = jl_typemap_assoc_by_type(mt->defs, tt, &env, inexact, 1, 0, world); + entry = jl_typemap_assoc_by_type(mt->defs, tt, &env, inexact, /*subtype*/1, /*offs*/0, world); if (entry == NULL || entry == INEXACT_ENTRY) { JL_GC_POP(); return NULL; @@ -1527,36 +1537,22 @@ jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs) return tt; } -jl_method_instance_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *types, - int cache, int inexact, int allow_exec, size_t world) +static jl_method_instance_t *jl_method_lookup_by_type( + jl_methtable_t *mt, jl_tupletype_t *types, + int cache, int inexact, int allow_exec, size_t world) { jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->cache, types, NULL, 0, 1, jl_cachearg_offset(mt), world); - if (entry) { + if (entry && entry->func.value) { jl_method_instance_t *linfo = (jl_method_instance_t*)entry->func.value; assert(linfo->min_world <= entry->min_world && linfo->max_world >= entry->max_world && "typemap consistency error: MethodInstance doesn't apply to full range of its entry"); return linfo; } JL_LOCK(&mt->writelock); - entry = jl_typemap_assoc_by_type(mt->cache, types, NULL, 0, 1, jl_cachearg_offset(mt), world); - if (entry) { - jl_method_instance_t *linfo = (jl_method_instance_t*)entry->func.value; - assert(linfo->min_world <= entry->min_world && linfo->max_world >= entry->max_world && - "typemap consistency error: MethodInstance doesn't apply to full range of its entry"); - JL_UNLOCK(&mt->writelock); - return linfo; - } if (jl_is_leaf_type((jl_value_t*)types)) cache = 1; jl_method_instance_t *sf = jl_mt_assoc_by_type(mt, types, cache, inexact, allow_exec, world); - if (cache) { - JL_UNLOCK(&mt->writelock); - } - else { - JL_GC_PUSH1(&sf); - JL_UNLOCK(&mt->writelock); - JL_GC_POP(); - } + JL_UNLOCK(&mt->writelock); return sf; } @@ -1571,24 +1567,11 @@ jl_method_instance_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, si if (entry) return entry->func.linfo; JL_LOCK(&mt->writelock); - entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt), world); - if (entry) { - JL_UNLOCK(&mt->writelock); - return entry->func.linfo; - } jl_tupletype_t *tt = arg_type_tuple(args, nargs); - jl_method_instance_t *sf = NULL; - JL_GC_PUSH2(&tt, &sf); - sf = jl_mt_assoc_by_type(mt, tt, cache, 0, 1, world); - if (cache) { - JL_UNLOCK(&mt->writelock); - } - else { - JL_GC_PUSH1(&sf); - JL_UNLOCK(&mt->writelock); - JL_GC_POP(); - } + JL_GC_PUSH1(&tt); + jl_method_instance_t *sf = jl_mt_assoc_by_type(mt, tt, cache, /*inexect*/0, 1, world); JL_GC_POP(); + JL_UNLOCK(&mt->writelock); return sf; } @@ -1882,18 +1865,12 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 } else { JL_LOCK(&mt->writelock); - entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt), world); - if (entry) { - mfunc = entry->func.linfo; - } - else { - // cache miss case - JL_TIMING(METHOD_LOOKUP_SLOW); - jl_tupletype_t *tt = arg_type_tuple(args, nargs); - JL_GC_PUSH1(&tt); - mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0, 1, world); - JL_GC_POP(); - } + // cache miss case + JL_TIMING(METHOD_LOOKUP_SLOW); + jl_tupletype_t *tt = arg_type_tuple(args, nargs); + JL_GC_PUSH1(&tt); + mfunc = jl_mt_assoc_by_type(mt, tt, /*cache*/1, /*inexect*/0, /*allow_exec*/1, world); + JL_GC_POP(); JL_UNLOCK(&mt->writelock); if (mfunc == NULL) { #ifdef JL_TRACE @@ -1977,26 +1954,19 @@ jl_value_t *jl_gf_invoke(jl_tupletype_t *types0, jl_value_t **args, size_t nargs } else { JL_LOCK(&method->writelock); - if (method->invokes.unknown != NULL) - tm = jl_typemap_assoc_exact(method->invokes, args, nargs, jl_cachearg_offset(mt), world); - if (tm) { - mfunc = tm->func.linfo; + tt = arg_type_tuple(args, nargs); + if (jl_is_unionall(entry->sig)) { + jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)entry->sig, &tpenv); + assert(ti != (jl_value_t*)jl_bottom_type); + (void)ti; } - else { - tt = arg_type_tuple(args, nargs); - if (jl_is_unionall(entry->sig)) { - jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)entry->sig, &tpenv); - assert(ti != (jl_value_t*)jl_bottom_type); - (void)ti; - } - sig = join_tsig(tt, entry->sig); - jl_method_t *func = entry->func.method; + sig = join_tsig(tt, entry->sig); + jl_method_t *func = entry->func.method; - if (func->invokes.unknown == NULL) - func->invokes.unknown = jl_nothing; + if (func->invokes.unknown == NULL) + func->invokes.unknown = jl_nothing; - mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, world, tpenv, 1); - } + mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, world, tpenv, 1); JL_UNLOCK(&method->writelock); } JL_GC_POP(); diff --git a/src/julia_internal.h b/src/julia_internal.h index 573d80125dd9d..14010c41950bf 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -474,8 +474,6 @@ jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e, int jl_is_toplevel_only_expr(jl_value_t *e); jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr); -jl_method_instance_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *types, - int cache, int inexact, int allow_exec, size_t world); jl_method_instance_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache, size_t world); jl_value_t *jl_gf_invoke(jl_tupletype_t *types, jl_value_t **args, size_t nargs); jl_method_instance_t *jl_lookup_generic(jl_value_t **args, uint32_t nargs, uint32_t callsite, size_t world); From 37bb40d6a34b1c98a7552bf2adb3caca2137cb9b Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 20 Jul 2017 13:15:33 -0400 Subject: [PATCH 10/12] fix IO deadlock condition when calling uv_shutdown on a handle being written to by another process we might never get the UV__POLLOUT notification but we also don't need to delay the uv_close call if we have nothing to write in the future, we should introduce a new `uv_drain` callback, instead of continuing to abuse uv_shutdown for this purpose fix #22832 Ref #22886 (cherry picked from commit 8949a95d44456fc90c376f344dbf5fcfdb69e789) --- src/jl_uv.c | 12 +++++++----- test/spawn.jl | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/jl_uv.c b/src/jl_uv.c index 3afc2c6cd6b25..894903e1db48c 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -197,16 +197,18 @@ JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) } if (handle->type == UV_NAMED_PIPE || handle->type == UV_TCP) { + uv_stream_t *stream = (uv_stream_t*)handle; #ifdef _OS_WINDOWS_ - if (((uv_stream_t*)handle)->stream.conn.shutdown_req) { + if (stream->stream.conn.shutdown_req) { #else - if (((uv_stream_t*)handle)->shutdown_req) { + if (stream->shutdown_req) { #endif // don't close the stream while attempting a graceful shutdown return; } - if (uv_is_writable((uv_stream_t*)handle)) { + if (uv_is_writable(stream) && stream->write_queue_size != 0) { // attempt graceful shutdown of writable streams to give them a chance to flush first + // TODO: introduce a uv_drain cb API instead of abusing uv_shutdown in this way uv_shutdown_t *req = (uv_shutdown_t*)malloc(sizeof(uv_shutdown_t)); req->data = 0; /* @@ -218,12 +220,12 @@ JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) * b) In case the stream is already closed, in which case uv_close would * cause an assertion failure. */ - uv_shutdown(req, (uv_stream_t*)handle, &jl_uv_shutdownCallback); + uv_shutdown(req, stream, &jl_uv_shutdownCallback); return; } } - if (!uv_is_closing((uv_handle_t*)handle)) { + if (!uv_is_closing(handle)) { // avoid double-closing the stream if (handle->type == UV_TTY) uv_tty_set_mode((uv_tty_t*)handle, UV_TTY_MODE_NORMAL); diff --git a/test/spawn.jl b/test/spawn.jl index f6c514c21bc1a..8ab298f337ea6 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -298,15 +298,25 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test iswritable(out) close(out.in) @test !isopen(out.in) - is_windows() || @test !isopen(out.out) # it takes longer to propagate EOF through the Windows event system - @test_throws ArgumentError write(out, "now closed error") - @test isreadable(out) @test !iswritable(out) + if !is_windows() + # on UNIX, we expect the pipe buffer is big enough that the write queue was immediately emptied + # and so we should already be notified of EPIPE on out.out by now + # and the other task should have already managed to consume all of the output + # it takes longer to propagate EOF through the Windows event system + # since it appears to be unwilling to buffer as much data + @test !isopen(out.out) + @test !isreadable(out) + end + @test_throws ArgumentError write(out, "now closed error") if is_windows() - # WINNT kernel does not provide a fast mechanism for async propagation + # WINNT kernel appears to not provide a fast mechanism for async propagation # of EOF for a blocking stream, so just wait for it to catch up. # This shouldn't take much more than 32ms. Base.wait_close(out) + # it's closed now, but the other task is expected to be behind this task + # in emptying the read buffer + @test isreadable(out) end @test !isopen(out) end @@ -466,3 +476,19 @@ end Base.showerror(io::IO, e::Error19864) = print(io, "correct19864") throw(Error19864())'`), stderr=catcmd)) == "ERROR: correct19864" + +## Deadlock in spawning a cmd (#22832) +# FIXME? +#let stdout = Pipe(), stdin = Pipe() +# Base.link_pipe(stdout, julia_only_read=true) +# Base.link_pipe(stdin, julia_only_write=true) +# p = spawn(pipeline(catcmd, stdin=stdin, stdout=stdout, stderr=DevNull)) +# @async begin # feed cat with 2 MB of data (zeros) +# write(stdin, zeros(UInt8, 1048576 * 2)) +# close(stdin) +# end +# sleep(0.5) # give cat a chance to fill the write buffer for stdout +# close(stdout.in) # make sure we can still close the write end +# @test sizeof(readstring(stdout)) == 1048576 * 2 # make sure we get all the data +# @test success(p) +#end From 7e4fa52f520078a94ca0287111d810e9fd3b7df9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Jun 2018 18:31:45 -0400 Subject: [PATCH 11/12] spawn test: attempt to enable test for #22832 (#27507) let's hope it passes and doesn't just hang... (cherry picked from commit 04456b9afba742d9ccd0d26185cece02dae2b796) --- test/spawn.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index 8ab298f337ea6..41ff152a98aff 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -478,17 +478,17 @@ end stderr=catcmd)) == "ERROR: correct19864" ## Deadlock in spawning a cmd (#22832) -# FIXME? -#let stdout = Pipe(), stdin = Pipe() -# Base.link_pipe(stdout, julia_only_read=true) -# Base.link_pipe(stdin, julia_only_write=true) -# p = spawn(pipeline(catcmd, stdin=stdin, stdout=stdout, stderr=DevNull)) -# @async begin # feed cat with 2 MB of data (zeros) -# write(stdin, zeros(UInt8, 1048576 * 2)) -# close(stdin) -# end -# sleep(0.5) # give cat a chance to fill the write buffer for stdout -# close(stdout.in) # make sure we can still close the write end -# @test sizeof(readstring(stdout)) == 1048576 * 2 # make sure we get all the data -# @test success(p) -#end +let out = Pipe(), inpt = Pipe() + Base.link_pipe(out, julia_only_read=true) + Base.link_pipe(inpt, julia_only_write=true) + p = spawn(pipeline(catcmd, stdin=inpt, stdout=out, stderr=DevNull)) + @async begin # feed cat with 2 MB of data (zeros) + write(inpt, zeros(UInt8, 1048576 * 2)) + close(inpt) + end + sleep(1) # give cat a chance to fill the write buffer for stdout + close(inpt.out) + close(out.in) # make sure we can still close the write end + @test sizeof(read(out)) == 1048576 * 2 # make sure we get all the data + @test success(p) +end From 156b0d7325fe737a52ac7546b042b2fd581993b9 Mon Sep 17 00:00:00 2001 From: Mus M Date: Sun, 16 Jul 2017 06:19:45 -0400 Subject: [PATCH 12/12] Fix exp10 fallback and exp2 and exp10 for Float16 (#22716) * Fix exp10 fallback and exp2 and exp10 for Float16 * Update docstring and test * Change float struct name * Update math.jl (cherry picked from commit 3ff6eac505746f78b018b6e3f175efebec574e2d) --- base/math.jl | 19 +++++++++++++++++-- test/math.jl | 25 ++++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/base/math.jl b/base/math.jl index dc00a548947ec..48369391d5919 100644 --- a/base/math.jl +++ b/base/math.jl @@ -262,14 +262,29 @@ cbrt(x::AbstractFloat) = x < 0 ? -(-x)^(1//3) : x^(1//3) """ exp2(x) -Compute ``2^x``. +Compute the base 2 exponential of `x`, in other words ``2^x``. +# Examples ```jldoctest julia> exp2(5) 32.0 ``` """ exp2(x::AbstractFloat) = 2^x + +""" + exp10(x) + +Compute the base 10 exponential of `x`, in other words ``10^x``. + +# Examples +```jldoctest +julia> exp10(2) +100.0 +``` +""" +exp10(x::AbstractFloat) = 10^x + for f in (:sinh, :cosh, :tanh, :atan, :asinh, :exp, :expm1) @eval ($f)(x::AbstractFloat) = error("not implemented for ", typeof(x)) end @@ -945,7 +960,7 @@ muladd(x,y,z) = x*y+z # Float16 definitions for func in (:sin,:cos,:tan,:asin,:acos,:atan,:sinh,:cosh,:tanh,:asinh,:acosh, - :atanh,:exp,:log,:log2,:log10,:sqrt,:lgamma,:log1p) + :atanh,:exp,:exp2,:exp10,:log,:log2,:log10,:sqrt,:lgamma,:log1p) @eval begin $func(a::Float16) = Float16($func(Float32(a))) $func(a::Complex32) = Complex32($func(Complex64(a))) diff --git a/test/math.jl b/test/math.jl index 04ce580e47346..e5ea6943b0bdf 100644 --- a/test/math.jl +++ b/test/math.jl @@ -246,11 +246,6 @@ end end end end -@test exp10(5) ≈ exp10(5.0) -@test exp10(50//10) ≈ exp10(5.0) -@test log10(exp10(e)) ≈ e -@test exp2(Float16(2.)) ≈ exp2(2.) -@test log(e) == 1 @testset "exp function" for T in (Float64, Float32) @testset "$T accuracy" begin @@ -614,3 +609,23 @@ end @testset "promote Float16 irrational #15359" begin @test typeof(Float16(.5) * pi) == Float16 end + +@testset "test fallback definitions" begin + @test exp10(5) ≈ exp10(5.0) + @test exp10(50//10) ≈ exp10(5.0) + @test log10(exp10(e)) ≈ e + @test log(e) === 1 + @test exp2(Float16(2.0)) ≈ exp2(2.0) + @test exp2(Float16(1.0)) === Float16(exp2(1.0)) + @test exp10(Float16(1.0)) === Float16(exp10(1.0)) +end + +# test AbstractFloat fallback pr22716 +struct Float22716{T<:AbstractFloat} <: AbstractFloat + x::T +end +Base.:^(x::Number, y::Float22716) = x^(y.x) +let x = 2.0 + @test exp2(Float22716(x)) === 2^x + @test exp10(Float22716(x)) === 10^x +end