GPU example throws errors #608
Comments
|
The workaround of course is to just convert directly to a CuArray: using DiffEqFlux, OrdinaryDiffEq, Flux, Optim, Plots, CUDA, DiffEqSensitivity
CUDA.allowscalar(false) # Makes sure no slow operations are occuring
# Generate Data
u0 = Float32[2.0; 0.0]
datasize = 30
tspan = (0.0f0, 1.5f0)
tsteps = range(tspan[1], tspan[2], length = datasize)
function trueODEfunc(du, u, p, t)
true_A = [-0.1 2.0; -2.0 -0.1]
du .= ((u.^3)'true_A)'
end
prob_trueode = ODEProblem(trueODEfunc, u0, tspan)
# Make the data into a GPU-based array if the user has a GPU
ode_data = CuArray(solve(prob_trueode, Tsit5(), saveat = tsteps))
dudt2 = FastChain((x, p) -> x.^3,
FastDense(2, 50, tanh),
FastDense(50, 2))
u0 = Float32[2.0; 0.0] |> gpu
p = initial_params(dudt2) |> gpu
prob_neuralode = NeuralODE(dudt2, tspan, Tsit5(), saveat = tsteps)
function predict_neuralode(p)
CuArray(prob_neuralode(u0,p))
end
function loss_neuralode(p)
pred = predict_neuralode(p)
loss = sum(abs2, ode_data .- pred)
return loss, pred
end
# Callback function to observe training
list_plots = []
iter = 0
callback = function (p, l, pred; doplot = false)
global list_plots, iter
if iter == 0
list_plots = []
end
iter += 1
display(l)
# plot current prediction against data
plt = scatter(tsteps, Array(ode_data[1,:]), label = "data")
scatter!(plt, tsteps, Array(pred[1,:]), label = "prediction")
push!(list_plots, plt)
if doplot
display(plot(plt))
end
return false
end
result_neuralode = DiffEqFlux.sciml_train(loss_neuralode, p,
ADAM(0.05), cb = callback,
maxiters = 300)@DhairyaLGandhi @mcabbott do you know why using RecursiveArrayTools, CUDA
x = VectorOfArray([rand(Float32,4),rand(Float32,4)])
CuArray(x) # works
gpu(x) # failsand it used to just call |
|
What's the zygote version? We haven't done a functors release but zygote was released recently. I'm not sure if that would explain it entirely, but surely this is a bug. Is it a recent RecursiveArrayTools release? |
|
Found the issue - it stems from the new CUDA v3.4 release series. with CUDA v3.4.1 x = VectorOfArray([rand(Float32,4),rand(Float32,4)]);
julia> Flux.CUDA.cu(x)
ERROR: MethodError: Cannot `convert` an object of type Vector{Float32} to an object of type Float32
Closest candidates are:
convert(::Type{T}, ::ColorTypes.Gray24) where T<:Real at /home/dhairyalgandhi/.julia/packages/ColorTypes/6m8P7/src/conversions.jl:114
convert(::Type{T}, ::ColorTypes.Gray) where T<:Real at /home/dhairyalgandhi/.julia/packages/ColorTypes/6m8P7/src/conversions.jl:113
convert(::Type{T}, ::Static.StaticFloat64{N}) where {N, T<:AbstractFloat} at /home/dhairyalgandhi/.julia/packages/Static/lCOFN/src/float.jl:26with CUDA v3.3.6 julia> x = VectorOfArray([rand(Float32,4),rand(Float32,4)]);
julia> Flux.CUDA.cu(x)
4×2 CUDA.CuArray{Float32, 2}:
0.840366 0.391562
0.682369 0.146678
0.15754 0.320327
0.0135089 0.0619746@maleadt have we changed something in |
|
Thanks! Reverting to CUDA v3.3.6 allows me to run that example on GPU. However, there are still issues with other functions, e.g. normalizing flows on GPU: gives the following error |
|
That trace suggests that something isn't on the gpu notice the |
|
Yeah, any issues with ffjord is probably a separate issue. Let's solve this in this issue ASAP and keep ffjord for another day. |
|
Regarding the VectorOfArray issue: Calling |
|
How did it work until the update though? Was https://github.com/SciML/RecursiveArrayTools.jl/blob/master/src/init.jl#L23-L30 |
|
Base.Array{U}(VA::AbstractVectorOfArray{T,N,A}) where {T,U,N,A <: AbstractVector{<:AbstractVector}} = reduce(hcat,map(x->U.(x), VA.u)) |
|
Yeah, I was just having trouble figuring out which overload was missing, so I was trying to track down what the change was. That seems to fix it all locally, so the new patch should handle this fine. Thanks @ccasert ! |
|
Commenting here from the Flux issue, could it be that JuliaGPU/GPUArrays.jl#368 (which was in GPUArrays 8/CUDA 3.4 but not GPUArrays 7/CUDA 3.3) changed things? |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I'm running into issues when trying out the following GPU example: https://diffeqflux.sciml.ai/dev/GPUs/.
At the line
I get this error
These are my package versions:
and this is my CUDA version:
The code works without any errors when running it on CPU. What could be the problem here?
Thanks!
The text was updated successfully, but these errors were encountered: