Segfault in __gmpn_mul_basecase_zen on 1.8

[KristofferC]

https://s3.amazonaws.com/julialang-reports/nanosoldier/pkgeval/by_hash/8b2e406_vs_742b9ab/RankRevealing.primary.log

The code below works when run in the Debugger but crashes when run with compiled mode. Works on 1.7.

julia> using Debugger, RankRevealing

julia> A = Matrix{Rational{BigInt}}(undef, 0,1 );

julia> B = [big"1" // big"1";;];

julia> @run grr(A, B) # works
GeneralizedRankRevealing{Rational{BigInt}, Matrix{Rational{BigInt}}}(Matrix{Rational{BigInt}}(undef, 0, 0), Rational{BigInt}[1//1;;], Rational{BigInt}[1//1;;])

julia> grr(A, B)
signal (11): Segmentation fault
in expression starting at REPL[9]:1
__gmpn_mul_basecase_zen at /home/kristofferc/julia/usr/bin/../lib/libgmp.so (unknown line)
Allocations: 52555958 (Pool: 52529405; Big: 26553); GC: 53
Segmentation fault (core dumped)

I'll try reducing it a bit more and see if someone is doing something fishy.

[KristofferC]

Creduce gives the following repo:

using LinearAlgebra

function hsplit(b, c)
  view(b, :, 1:c), b  
end

struct PLUQ{a, b }
  c  
  rank      
  factors  end
  function PLUQ(c , d, e::a) where a
  PLUQ{a,o}(vec, d, e)
end
function Base.getproperty(f::PLUQ, key::Symbol)  
  if key == :g
    getfield(f, :factors)
  elseif key == :i
    d = getfield(f, :rank)
    getfield(f, :factors)[d+1:end ]
  else
    getfield(f, key)
  end
end
Base.iterate(b::PLUQ)            = b 
Base.iterate(b::PLUQ, :) = b.g, Val(:)
Base.iterate(b::PLUQ, ::Val) = b.g, Val(:h)
Base.iterate(b::PLUQ, ::Val{:h}) = b, Val(:i)
Base.iterate(b::PLUQ, ::Val{:i}) = b.i, 0

function f(b)
  h, j, k, g, m = PLUQ([], 1, b)
   m,  k
end
function o(b) 
  i = [I ;  b]
  i, i
end
function grr(b, s)
  m, t    = f([b ; s])
  a, p   = view(m, : ), view(m, : )
  v, ab   = o(p)
  ac       = ab / ab
  ad, ae = hsplit(ac, - size(v,2))
  ai = view(ae, : )
  _, al   = o(ai)
  am = zeros(Int64, size(al, 1), size(ad,2))
  [ad ai ; am al] * t
end

A = Matrix{Rational{BigInt}}(undef, 0,1 )
B = big"1"
grr(A, B)

[vtjnash]

also seems to work on master already, so we may be able to bisect this? It is odd looking, inference thought we should have called (which would be correct): Inference on grr should have resolved * to be unknown (failing in vcat), since we expect to call * on two Matrix{Rational{BigInt}}

invoke _generic_matmatmul!(::Matrix{Rational{BigInt}},::Char,::Char,::Matrix{Rational{BigInt}},::Matrix{Rational{BigInt}},::LinearAlgebra.MulAddMul{true, true, Bool, Bool})::Matrix{Rational{BigInt}}

but we actually assumed it was calling * between Matrix{BigFloat} and Matrix{Rational{BigInt}}:

_generic_matmatmul!(Array{Base.MPFR.BigFloat, 2}, Char, Char, Array{Base.MPFR.BigFloat, 2}, Array{Base.Rational{Base.GMP.BigInt}, 2}, LinearAlgebra.MulAddMul{true, true, Bool, Bool})

and MPFR doesn't like that we had a BigInt where there should have been a BigFloat in mpfr_mul

[vtjnash]

master (correct)

hvcat(rows::Tuple{Vararg{Int64}}, xs::AbstractVecOrMat...) in Base at abstractarray.jl:2008
Variables
  #self#::Core.Const(hvcat)
  rows::Tuple{Int64, Int64}
  xs::Tuple{Union{SubArray{Rational{BigInt}, 2, Matrix{Rational{BigInt}}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}, SubArray{BigFloat, 2, Matrix{BigFloat}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}}, Union{SubArray{Rational{BigInt}, 1, Vector{Rational{BigInt}}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}, SubArray{BigFloat, 1, Vector{BigFloat}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}}, Matrix{Int64}, Union{Matrix{Rational{BigInt}}, Matrix{BigFloat}}}

│ ─ %-1  = invoke hvcat(…,…,…,…,…)::Matrix
    @ abstractarray.jl:2008 within `hvcat`
1 ─ %1 = Core._apply_iterate(Base.iterate, Base.promote_eltype, xs)::Union{Type{Rational{BigInt}}, Type{BigFloat}}
│   %2 = Core.tuple(%1, rows)::Tuple{Type, Tuple{Int64, Int64}}
│   %3 = Core._apply_iterate(Base.iterate, Base.typed_hvcat, %2, xs)::Matrix
└──      return %3

v1.8 (incorrect)

hvcat(rows::Tuple{Vararg{Int64}}, xs::AbstractVecOrMat...) in Base at abstractarray.jl:1987
Variables
  #self#::Core.Const(hvcat)
  rows::Tuple{Int64, Int64}
  xs::Tuple{Union{SubArray{Rational{BigInt}, 2, Matrix{Rational{BigInt}}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}, SubArray{BigFloat, 2, Matrix{BigFloat}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}}, Union{SubArray{Rational{BigInt}, 1, Vector{Rational{BigInt}}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}, SubArray{BigFloat, 1, Vector{BigFloat}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}}, Matrix{Int64}, Union{Matrix{Rational{BigInt}}, Matrix{BigFloat}}}

│ ─ %-1  = invoke hvcat(…,…,…,…,…)::Matrix{BigFloat}
    @ abstractarray.jl:1987 within `hvcat`
1 ─ %1 = Core._apply_iterate(Base.iterate, Base.promote_eltype, xs)::Union{Type{Rational{BigInt}}, Type{BigFloat}}
│   %2 = Core.tuple(%1, rows)::Tuple{Union{Type{Rational{BigInt}}, Type{BigFloat}}, Tuple{Int64, Int64}}
│   %3 = Core._apply_iterate(Base.iterate, Base.typed_hvcat, %2, xs)::Matrix{BigFloat}
└──      return %3

This bug is probably identical to #44705