Backports for 1.9.0-rc2

Make.inc

@@ -89,6 +89,9 @@ WITH_GC_DEBUG_ENV := 0
 # Enable DTrace support
 WITH_DTRACE := 0
 
+# Enable ITTAPI integration
+WITH_ITTAPI := 0
+
 # Prevent picking up $ARCH from the environment variables
 ARCH:=
 
@@ -303,6 +306,9 @@ private_libdir := $(libdir)/julia
 endif
 build_private_libdir := $(build_libdir)/julia
 
+private_libexecdir := $(libexecdir)/julia
+build_private_libexecdir := $(build_libexecdir)/julia
+
 # A helper functions for dealing with lazily-evaluated, expensive operations.. Spinning
 # up a python process to, for exaxmple, parse a TOML file is expensive, and we must wait
 # until the TOML files are on-disk before we can parse them. This means that we cannot
@@ -327,7 +333,7 @@ define cache_rel_path
 $(1)_rel_eval = $(call rel_path,$(2),$($(1)))
 $(1)_rel = $$(call hit_cache,$(1)_rel_eval)
 endef
-$(foreach D,libdir private_libdir datarootdir libexecdir docdir sysconfdir includedir,$(eval $(call cache_rel_path,$(D),$(bindir))))
+$(foreach D,libdir private_libdir datarootdir libexecdir private_libexecdir docdir sysconfdir includedir,$(eval $(call cache_rel_path,$(D),$(bindir))))
 $(foreach D,build_libdir build_private_libdir,$(eval $(call cache_rel_path,$(D),$(build_bindir))))
 
 # Save a special one: reverse_private_libdir_rel: usually just `../`, but good to be general:
@@ -733,7 +739,12 @@ ifeq ($(WITH_DTRACE), 1)
 JCXXFLAGS += -DUSE_DTRACE
 JCFLAGS += -DUSE_DTRACE
 DTRACE := dtrace
-else
+endif
+
+ifeq ($(WITH_ITTAPI), 1)
+JCXXFLAGS += -DUSE_ITTAPI
+JCFLAGS += -DUSE_ITTAPI
+LIBITTAPI:=-littnotify
 endif
 
 # ===========================================================================

Makefile

@@ -238,7 +238,7 @@ endef
 
 install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
  @$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE)
- @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \
+ @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(private_libexecdir); do \
  mkdir -p $(DESTDIR)$$subdir; \
  done
 
@@ -253,8 +253,8 @@ else ifeq ($(JULIA_BUILD_MODE),debug)
  -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/
 endif
 
- # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it.
- -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/
+ # We have a single exception; we want 7z.dll to live in private_libexecdir, not bindir, so that 7z.exe can find it.
+ -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(private_libexecdir)/
  -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
  -$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/
  # The rest are compiler dependencies, as an example memcpy is exported by msvcrt
@@ -311,14 +311,14 @@ endif
  done \
  done
 endif
- # Install `7z` into libexec/
- $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(libexecdir)/
+ # Install `7z` into private_libexecdir
+ $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(private_libexecdir)/
 
- # Install `lld` into libexec/
- $(INSTALL_M) $(build_depsbindir)/lld$(EXE) $(DESTDIR)$(libexecdir)/
+ # Install `lld` into private_libexecdir
+ $(INSTALL_M) $(build_depsbindir)/lld$(EXE) $(DESTDIR)$(private_libexecdir)/
 
- # Install `dsymutil` into libexec/
- $(INSTALL_M) $(build_depsbindir)/dsymutil$(EXE) $(DESTDIR)$(libexecdir)/
+ # Install `dsymutil` into private_libexecdir/
+ $(INSTALL_M) $(build_depsbindir)/dsymutil$(EXE) $(DESTDIR)$(private_libexecdir)/
 
  # Copy public headers
  cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia

THIRDPARTY.md

@@ -24,6 +24,10 @@ own licenses:
 - [LLVM](https://releases.llvm.org/12.0.1/LICENSE.TXT) [APACHE 2.0 with LLVM Exception]
 - [UTF8PROC](https://github.com/JuliaStrings/utf8proc) [MIT]
 
+and optionally:
+
+- [ITTAPI](https://github.com/intel/ittapi/blob/master/LICENSES/BSD-3-Clause.txt) [BSD-3]
+
 Julia's `stdlib` uses the following external libraries, which have their own licenses:
 
 - [DSFMT](https://github.com/MersenneTwister-Lab/dSFMT/blob/master/LICENSE.txt) [BSD-3]

base/Makefile

@@ -66,6 +66,7 @@ ifeq ($(OS),WINNT)
  @printf 'const LIBDIR = "%s"\n' '$(subst /,\\,$(libdir_rel))' >> $@
  @printf 'const LIBEXECDIR = "%s"\n' '$(subst /,\\,$(libexecdir_rel))' >> $@
  @printf 'const PRIVATE_LIBDIR = "%s"\n' '$(subst /,\\,$(private_libdir_rel))' >> $@
+ @printf 'const PRIVATE_LIBEXECDIR = "%s"\n' '$(subst /,\\,$(private_libexecdir_rel))' >> $@
  @printf 'const INCLUDEDIR = "%s"\n' '$(subst /,\\,$(includedir_rel))' >> $@
 else
  @echo "const SYSCONFDIR = \"$(sysconfdir_rel)\"" >> $@
@@ -74,6 +75,7 @@ else
  @echo "const LIBDIR = \"$(libdir_rel)\"" >> $@
  @echo "const LIBEXECDIR = \"$(libexecdir_rel)\"" >> $@
  @echo "const PRIVATE_LIBDIR = \"$(private_libdir_rel)\"" >> $@
+ @echo "const PRIVATE_LIBEXECDIR = \"$(private_libexecdir_rel)\"" >> $@
  @echo "const INCLUDEDIR = \"$(includedir_rel)\"" >> $@
 endif
 ifeq ($(DARWIN_FRAMEWORK), 1)

base/abstractarray.jl

@@ -1982,12 +1982,16 @@ julia> cat(1, [2], [3;;]; dims=Val(2))
 
 # The specializations for 1 and 2 inputs are important
 # especially when running with --inline=no, see #11158
+# The specializations for Union{AbstractVecOrMat,Number} are necessary
+# to have more specialized methods here than in LinearAlgebra/uniformscaling.jl
 vcat(A::AbstractArray) = cat(A; dims=Val(1))
 vcat(A::AbstractArray, B::AbstractArray) = cat(A, B; dims=Val(1))
 vcat(A::AbstractArray...) = cat(A...; dims=Val(1))
+vcat(A::Union{AbstractVecOrMat,Number}...) = cat(A...; dims=Val(1))
 hcat(A::AbstractArray) = cat(A; dims=Val(2))
 hcat(A::AbstractArray, B::AbstractArray) = cat(A, B; dims=Val(2))
 hcat(A::AbstractArray...) = cat(A...; dims=Val(2))
+hcat(A::Union{AbstractVecOrMat,Number}...) = cat(A...; dims=Val(2))
 
 typed_vcat(T::Type, A::AbstractArray) = _cat_t(Val(1), T, A)
 typed_vcat(T::Type, A::AbstractArray, B::AbstractArray) = _cat_t(Val(1), T, A, B)
@@ -2137,6 +2141,8 @@ end
 
 hvcat(rows::Tuple{Vararg{Int}}, xs::Number...) = typed_hvcat(promote_typeof(xs...), rows, xs...)
 hvcat(rows::Tuple{Vararg{Int}}, xs...) = typed_hvcat(promote_eltypeof(xs...), rows, xs...)
+# the following method is needed to provide a more specific one compared to LinearAlgebra/uniformscaling.jl
+hvcat(rows::Tuple{Vararg{Int}}, xs::Union{AbstractVecOrMat,Number}...) = typed_hvcat(promote_eltypeof(xs...), rows, xs...)
 
 function typed_hvcat(::Type{T}, rows::Tuple{Vararg{Int}}, xs::Number...) where T
  nr = length(rows)

base/array.jl

@@ -177,11 +177,11 @@ function _unsetindex!(A::Array{T}, i::Int) where {T}
  t = @_gc_preserve_begin A
  p = Ptr{Ptr{Cvoid}}(pointer(A, i))
  if !allocatedinline(T)
- unsafe_store!(p, C_NULL)
+ Intrinsics.atomic_pointerset(p, C_NULL, :monotonic)
  elseif T isa DataType
  if !datatype_pointerfree(T)
- for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
- unsafe_store!(p, C_NULL, j)
+ for j = 1:Core.sizeof(Ptr{Cvoid}):Core.sizeof(T)
+ Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic)
  end
  end
  end
@@ -1916,7 +1916,7 @@ function reverse!(v::AbstractVector, start::Integer, stop::Integer=lastindex(v))
  return v
 end
 
-# concatenations of homogeneous combinations of vectors, horizontal and vertical
+# concatenations of (in)homogeneous combinations of vectors, horizontal and vertical
 
 vcat() = Vector{Any}()
 hcat() = Vector{Any}()
@@ -1930,6 +1930,7 @@ function hcat(V::Vector{T}...) where T
  end
  return [ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ]
 end
+hcat(A::Vector...) = cat(A...; dims=Val(2)) # more special than SparseArrays's hcat
 
 function vcat(arrays::Vector{T}...) where T
  n = 0
@@ -1946,6 +1947,19 @@ function vcat(arrays::Vector{T}...) where T
  end
  return arr
 end
+vcat(A::Vector...) = cat(A...; dims=Val(1)) # more special than SparseArrays's vcat
+
+# disambiguation with LinAlg/special.jl
+# Union{Number,Vector,Matrix} is for LinearAlgebra._DenseConcatGroup
+# VecOrMat{T} is for LinearAlgebra._TypedDenseConcatGroup
+hcat(A::Union{Number,Vector,Matrix}...) = cat(A...; dims=Val(2))
+hcat(A::VecOrMat{T}...) where {T} = typed_hcat(T, A...)
+vcat(A::Union{Number,Vector,Matrix}...) = cat(A...; dims=Val(1))
+vcat(A::VecOrMat{T}...) where {T} = typed_vcat(T, A...)
+hvcat(rows::Tuple{Vararg{Int}}, xs::Union{Number,Vector,Matrix}...) =
+ typed_hvcat(promote_eltypeof(xs...), rows, xs...)
+hvcat(rows::Tuple{Vararg{Int}}, xs::VecOrMat{T}...) where {T} =
+ typed_hvcat(T, rows, xs...)
 
 _cat(n::Integer, x::Integer...) = reshape([x...], (ntuple(Returns(1), n-1)..., length(x)))
 

base/compiler/abstractinterpretation.jl

@@ -204,7 +204,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f),
  if seen ≠ napplicable
  # there is unanalyzed candidate, widen type and effects to the top
  rettype = Any
- all_effects = Effects()
+ # there may be unanalyzed effects within unseen dispatch candidate,
+ # but we can still ignore nonoverlayed effect here since we already accounted for it
+ all_effects = merge_effects(all_effects, EFFECTS_UNKNOWN)
  elseif isa(matches, MethodMatches) ? (!matches.fullmatch || any_ambig(matches)) :
  (!all(matches.fullmatches) || any_ambig(matches))
  # Account for the fact that we may encounter a MethodError with a non-covered or ambiguous signature.
@@ -1304,21 +1306,32 @@ function ssa_def_slot(@nospecialize(arg), sv::InferenceState)
  return arg
 end
 
+struct AbstractIterationResult
+ cti::Vector{Any}
+ info::MaybeAbstractIterationInfo
+ ai_effects::Effects
+end
+AbstractIterationResult(cti::Vector{Any}, info::MaybeAbstractIterationInfo) =
+ AbstractIterationResult(cti, info, EFFECTS_TOTAL)
+
 # `typ` is the inferred type for expression `arg`.
 # if the expression constructs a container (e.g. `svec(x,y,z)`),
 # refine its type to an array of element types.
 # Union of Tuples of the same length is converted to Tuple of Unions.
 # returns an array of types
 function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft), @nospecialize(typ),
  sv::Union{InferenceState, IRCode})
- if isa(typ, PartialStruct) && typ.typ.name === Tuple.name
- return typ.fields, nothing
+ if isa(typ, PartialStruct)
+ widet = typ.typ
+ if isa(widet, DataType) && widet.name === Tuple.name
+ return AbstractIterationResult(typ.fields, nothing)
+ end
  end
 
  if isa(typ, Const)
  val = typ.val
  if isa(val, SimpleVector) || isa(val, Tuple)
- return Any[ Const(val[i]) for i in 1:length(val) ], nothing # avoid making a tuple Generator here!
+ return AbstractIterationResult(Any[ Const(val[i]) for i in 1:length(val) ], nothing) # avoid making a tuple Generator here!
  end
  end
 
@@ -1333,12 +1346,12 @@ function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft)
  if isa(tti, Union)
  utis = uniontypes(tti)
  if any(@nospecialize(t) -> !isa(t, DataType) || !(t <: Tuple) || !isknownlength(t), utis)
- return Any[Vararg{Any}], nothing
+ return AbstractIterationResult(Any[Vararg{Any}], nothing, EFFECTS_UNKNOWN′)
  end
  ltp = length((utis[1]::DataType).parameters)
  for t in utis
  if length((t::DataType).parameters) != ltp
- return Any[Vararg{Any}], nothing
+ return AbstractIterationResult(Any[Vararg{Any}], nothing)
  end
  end
  result = Any[ Union{} for _ in 1:ltp ]
@@ -1349,12 +1362,12 @@ function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft)
  result[j] = tmerge(result[j], rewrap_unionall(tps[j], tti0))
  end
  end
- return result, nothing
+ return AbstractIterationResult(result, nothing)
  elseif tti0 <: Tuple
  if isa(tti0, DataType)
- return Any[ p for p in tti0.parameters ], nothing
+ return AbstractIterationResult(Any[ p for p in tti0.parameters ], nothing)
  elseif !isa(tti, DataType)
- return Any[Vararg{Any}], nothing
+ return AbstractIterationResult(Any[Vararg{Any}], nothing)
  else
  len = length(tti.parameters)
  last = tti.parameters[len]
@@ -1363,12 +1376,14 @@ function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft)
  if va
  elts[len] = Vararg{elts[len]}
  end
- return elts, nothing
+ return AbstractIterationResult(elts, nothing)
  end
- elseif tti0 === SimpleVector || tti0 === Any
- return Any[Vararg{Any}], nothing
+ elseif tti0 === SimpleVector
+ return AbstractIterationResult(Any[Vararg{Any}], nothing)
+ elseif tti0 === Any
+ return AbstractIterationResult(Any[Vararg{Any}], nothing, EFFECTS_UNKNOWN′)
  elseif tti0 <: Array
- return Any[Vararg{eltype(tti0)}], nothing
+ return AbstractIterationResult(Any[Vararg{eltype(tti0)}], nothing)
  else
  return abstract_iteration(interp, itft, typ, sv)
  end
@@ -1379,7 +1394,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  if isa(itft, Const)
  iteratef = itft.val
  else
- return Any[Vararg{Any}], nothing
+ return AbstractIterationResult(Any[Vararg{Any}], nothing, EFFECTS_UNKNOWN′)
  end
  @assert !isvarargtype(itertype)
  call = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[itft, itertype]), StmtInfo(true), sv)
@@ -1389,7 +1404,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  # WARNING: Changes to the iteration protocol must be reflected here,
  # this is not just an optimization.
  # TODO: this doesn't realize that Array, SimpleVector, Tuple, and NamedTuple do not use the iterate protocol
- stateordonet === Bottom && return Any[Bottom], AbstractIterationInfo(CallMeta[CallMeta(Bottom, call.effects, info)])
+ stateordonet === Bottom && return AbstractIterationResult(Any[Bottom], AbstractIterationInfo(CallMeta[CallMeta(Bottom, call.effects, info)], true))
  valtype = statetype = Bottom
  ret = Any[]
  calls = CallMeta[call]
@@ -1399,7 +1414,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  # length iterators, or interesting prefix
  while true
  if stateordonet_widened === Nothing
- return ret, AbstractIterationInfo(calls)
+ return AbstractIterationResult(ret, AbstractIterationInfo(calls, true))
  end
  if Nothing <: stateordonet_widened || length(ret) >= InferenceParams(interp).MAX_TUPLE_SPLAT
  break
@@ -1411,7 +1426,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  # If there's no new information in this statetype, don't bother continuing,
  # the iterator won't be finite.
  if ⊑(typeinf_lattice(interp), nstatetype, statetype)
- return Any[Bottom], nothing
+ return AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), EFFECTS_THROWS)
  end
  valtype = getfield_tfunc(typeinf_lattice(interp), stateordonet, Const(1))
  push!(ret, valtype)
@@ -1441,7 +1456,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  # ... but cannot terminate
  if !may_have_terminated
  # ... and cannot have terminated prior to this loop
- return Any[Bottom], nothing
+ return AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), EFFECTS_UNKNOWN′)
  else
  # iterator may have terminated prior to this loop, but not during it
  valtype = Bottom
@@ -1451,13 +1466,15 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n
  end
  valtype = tmerge(valtype, nounion.parameters[1])
  statetype = tmerge(statetype, nounion.parameters[2])
- stateordonet = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, statetype]), StmtInfo(true), sv).rt
+ call = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, statetype]), StmtInfo(true), sv)
+ push!(calls, call)
+ stateordonet = call.rt
  stateordonet_widened = widenconst(stateordonet)
  end
  if valtype !== Union{}
  push!(ret, Vararg{valtype})
  end
- return ret, nothing
+ return AbstractIterationResult(ret, AbstractIterationInfo(calls, false))
 end
 
 # do apply(af, fargs...), where af is a function value
@@ -1488,13 +1505,9 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si::
  infos′ = Vector{MaybeAbstractIterationInfo}[]
  for ti in (splitunions ? uniontypes(aargtypes[i]) : Any[aargtypes[i]])
  if !isvarargtype(ti)
- cti_info = precise_container_type(interp, itft, ti, sv)
- cti = cti_info[1]::Vector{Any}
- info = cti_info[2]::MaybeAbstractIterationInfo
+ (;cti, info, ai_effects) = precise_container_type(interp, itft, ti, sv)
  else
- cti_info = precise_container_type(interp, itft, unwrapva(ti), sv)
- cti = cti_info[1]::Vector{Any}
- info = cti_info[2]::MaybeAbstractIterationInfo
+ (;cti, info, ai_effects) = precise_container_type(interp, itft, unwrapva(ti), sv)
  # We can't represent a repeating sequence of the same types,
  # so tmerge everything together to get one type that represents
  # everything.
@@ -1507,6 +1520,12 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si::
  end
  cti = Any[Vararg{argt}]
  end
+ effects = merge_effects(effects, ai_effects)
+ if info !== nothing
+ for call in info.each
+ effects = merge_effects(effects, call.effects)
+ end
+ end
  if any(@nospecialize(t) -> t === Bottom, cti)
  continue
  end

base/compiler/inferencestate.jl

@@ -459,7 +459,18 @@ function sptypes_from_meth_instance(linfo::MethodInstance)
  v = sp[i]
  if v isa TypeVar
  fromArg = 0
- maybe_undef = !constrains_param(v, linfo.specTypes, #=covariant=#true)
+ maybe_undef = !(let sig=sig
+ # if the specialized signature `linfo.specTypes` doesn't contain any free
+ # type variables, we can use it for a more accurate analysis of whether `v`
+ # is constrained or not, otherwise we should use `def.sig` which always
+ # doesn't contain any free type variables
+ if !has_free_typevars(linfo.specTypes)
+ sig = linfo.specTypes
+ else
+ @assert !has_free_typevars(sig)
+ end
+ constrains_param(v, sig, #=covariant=#true)
+ end)
  temp = sig
  for j = 1:i-1
  temp = temp.body