while training, get InvalidIRError

[barcoyou]

julia> train_set = ((gpu(cat(train[batch]/255f0..., dims=4)), gpu(onehotbatch(label[batch], scores))) for batch in partition(randperm(length(train)), batch_size))
Base.Generator{Base.Iterators.PartitionIterator{Array{Int64,1}},getfield(Main, Symbol("##33#34"))}(getfield(Main, Symbol("##33#34"))(), Base.Iterators.PartitionIterator{Array{Int64,1}}([22, 19, 1, 37, 13, 10, 21, 15, 33, 6 … 26, 8, 11, 7, 5, 24, 20, 35, 16, 31], 3))

julia> Flux.train!(loss, params(model), train_set, opt, cb=evalcb)
ERROR: InvalidIRError: compiling mapreducedim_kernel_parallel(typeof(identity), typeof(Base.add_sum), CUDAnative.CuDeviceArray{Float32,2,CUDAnative.AS.Global}, CUDAnative.CuDeviceArray{Float32,2,CUDAnative.AS.Global}, CartesianIndices{2,Tuple{Base.OneTo{Int64},Base.OneTo{Int64}}}, Int64, Int64) resulted in invalid LLVM IR
Reason: unsupported dynamic function invocation (call to throw_inexacterror(f::Symbol, T, val) in Core at boot.jl:560)
Stacktrace:
[1] check_top_bit at boot.jl:574
[2] toInt64 at boot.jl:635
[3] Type at boot.jl:710
[4] convert at number.jl:7
[5] steprange_last at range.jl:225
[6] multiple call sites at unknown:0
Stacktrace:
[1] check_ir(::CUDAnative.CompilerJob, ::LLVM.Module) at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/compiler/validation.jl:114
[2] macro expansion at /home/dsd/.julia/packages/TimerOutputs/7zSea/src/TimerOutput.jl:216 [inlined]
[3] #codegen#119(::Bool, ::Bool, ::Bool, ::Bool, ::Bool, ::typeof(CUDAnative.codegen), ::Symbol, ::CUDAnative.CompilerJob) at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/compiler/driver.jl:186
[4] #codegen at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/compiler/driver.jl:0 [inlined]
[5] #compile#118(::Bool, ::Bool, ::Bool, ::Bool, ::Bool, ::typeof(CUDAnative.compile), ::Symbol, ::CUDAnative.CompilerJob) at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/compiler/driver.jl:47
[6] #compile#117 at ./none:0 [inlined]
[7] compile at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/compiler/driver.jl:28 [inlined] (repeats 2 times)
[8] macro expansion at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/execution.jl:378 [inlined]
[9] #cufunction#159(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.cufunction), ::typeof(CuArrays.mapreducedim_kernel_parallel), ::Type{Tuple{typeof(identity),typeof(Base.add_sum),CUDAnative.CuDeviceArray{Float32,2,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float32,2,CUDAnative.AS.Global},CartesianIndices{2,Tuple{Base.OneTo{Int64},Base.OneTo{Int64}}},Int64,Int64}}) at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/execution.jl:347
[10] cufunction(::Function, ::Type) at /home/dsd/.julia/packages/CUDAnative/RcKr9/src/execution.jl:347
[11] macro expansion at /home/dsd/.julia/packages/CuArrays/R5hMd/src/mapreduce.jl:68 [inlined]
[12] macro expansion at ./gcutils.jl:87 [inlined]
[13] mapreducedim!(::Function, ::Function, ::CuArray{Float32,2}, ::CuArray{Float32,2}) at /home/dsd/.julia/packages/CuArrays/R5hMd/src/mapreduce.jl:65
[14] mapreducedim! at ./reducedim.jl:274 [inlined]
[15] mapreduce_dim at ./reducedim.jl:317 [inlined]
[16] mapreduce_impl at /home/dsd/.julia/packages/GPUArrays/CjRPU/src/mapreduce.jl:73 [inlined]
[17] #mapreduce#48 at /home/dsd/.julia/packages/GPUArrays/CjRPU/src/mapreduce.jl:59 [inlined]
[18] #mapreduce at ./none:0 [inlined]
[19] sum at ./reducedim.jl:679 [inlined]
[20] sum at ./reducedim.jl:678 [inlined]
[21] #sum#559 at ./reducedim.jl:652 [inlined]
[22] #sum at ./none:0 [inlined]
[23] #forward#481 at /home/dsd/.julia/packages/Tracker/RRYy6/src/lib/array.jl:315 [inlined]
[24] #forward at ./none:0 [inlined]
[25] #track#1 at /home/dsd/.julia/packages/Tracker/RRYy6/src/Tracker.jl:51 [inlined]
[26] #track at ./none:0 [inlined]
[27] #sum#480 at /home/dsd/.julia/packages/Tracker/RRYy6/src/lib/array.jl:312 [inlined]
[28] #sum at ./none:0 [inlined]
[29] unbroadcast(::TrackedArray{…,CuArray{Float32,1}}, ::TrackedArray{…,CuArray{Float32,2}}) at /home/dsd/.julia/packages/Tracker/RRYy6/src/lib/array.jl:467
[30] back(::Tracker.Call{getfield(Tracker, Symbol("#back#548")){2,getfield(Base.Broadcast, Symbol("##2#4")){getfield(Base.Broadcast, Symbol("##8#10")){getfield(Base.Broadcast, Symbol("##1#3")),getfield(Base.Broadcast, Symbol("##5#6")){getfield(Base.Broadcast, Symbol("##5#6")){getfield(Base.Broadcast, Symbol("##7#9"))}},getfield(Base.Broadcast, Symbol("##11#12")){getfield(Base.Broadcast, Symbol("##11#12")){getfield(Base.Broadcast, Symbol("##13#14"))}},getfield(Base.Broadcast, Symbol("##15#16")){getfield(Base.Broadcast, Symbol("##15#16")){getfield(Base.Broadcast, Symbol("##17#18"))}},typeof(+)},typeof(identity)},Tuple{TrackedArray{…,CuArray{Float32,2}},TrackedArray{…,CuArray{Float32,1}}}},Tuple{Tracker.Tracked{CuArray{Float32,2}},Tracker.Tracked{CuArray{Float32,1}}}}, ::CuArray{Float32,2}, ::Bool) at ./tuple.jl:159
[31] back(::Tracker.Tracked{CuArray{Float32,2}}, ::CuArray{Float32,2}, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:58
[32] foreach at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38 [inlined]
[33] back(::Tracker.Call{getfield(Tracker, Symbol("##511#512")){TrackedArray{…,CuArray{Float32,2}}},Tuple{Tracker.Tracked{CuArray{Float32,2}}}}, ::CuArray{Float32,2}, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38
[34] back(::Tracker.Tracked{CuArray{Float32,2}}, ::CuArray{Float32,2}, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:58
[35] #13 at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38 [inlined]
[36] foreach at ./abstractarray.jl:1921 [inlined]
... (the last 4 lines are repeated 1 more time)
[41] back(::Tracker.Call{getfield(Tracker, Symbol("##482#483")){TrackedArray{…,CuArray{Float32,2}}},Tuple{Tracker.Tracked{CuArray{Float32,2}}}}, ::Float32, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38
[42] back(::Tracker.Tracked{Float32}, ::Float32, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:58
[43] foreach at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38 [inlined]
[44] back(::Tracker.Call{getfield(Tracker, Symbol("##199#200")),Tuple{Tracker.Tracked{Float32}}}, ::Float32, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38
[45] back(::Tracker.Tracked{Float32}, ::Float32, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:58
[46] #13 at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38 [inlined]
[47] foreach at ./abstractarray.jl:1921 [inlined]
[48] back(::Tracker.Call{getfield(Tracker, Symbol("##278#281")){Rational{Int64}},Tuple{Tracker.Tracked{Float32},Nothing}}, ::Float32, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:38
[49] back(::Tracker.Tracked{Float32}, ::Int64, ::Bool) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:58
[50] #back!#15 at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:77 [inlined]
[51] #back! at ./none:0 [inlined]
[52] #back!#32 at /home/dsd/.julia/packages/Tracker/RRYy6/src/lib/real.jl:16 [inlined]
[53] back!(::Tracker.TrackedReal{Float32}) at /home/dsd/.julia/packages/Tracker/RRYy6/src/lib/real.jl:14
[54] gradient(::getfield(Flux.Optimise, Symbol("##15#21")){typeof(loss),Tuple{CuArray{Float32,4},Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1}}}}, ::Tracker.Params) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:4
[55] #gradient#24(::Bool, ::typeof(Tracker.gradient), ::Function, ::Tracker.Params) at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:164
[56] gradient at /home/dsd/.julia/packages/Tracker/RRYy6/src/back.jl:164 [inlined]
[57] macro expansion at /home/dsd/.julia/packages/Flux/3GyGP/src/optimise/train.jl:71 [inlined]
[58] macro expansion at /home/dsd/.julia/packages/Juno/TfNYn/src/progress.jl:133 [inlined]
[59] #train!#12(::getfield(Flux, Symbol("#throttled#14")){getfield(Flux, Symbol("##throttled#10#15")){Bool,Bool,getfield(Main, Symbol("##7#9")),Int64}}, ::typeof(Flux.Optimise.train!), ::Function, ::Tracker.Params, ::Base.Generator{Base.Iterators.PartitionIterator{Array{Int64,1}},getfield(Main, Symbol("##33#34"))}, ::ADAM) at /home/dsd/.julia/packages/Flux/3GyGP/src/optimise/train.jl:69
[60] (::getfield(Flux.Optimise, Symbol("#kw##train!")))(::NamedTuple{(:cb,),Tuple{getfield(Flux, Symbol("#throttled#14")){getfield(Flux, Symbol("##throttled#10#15")){Bool,Bool,getfield(Main, Symbol("##7#9")),Int64}}}}, ::typeof(Flux.Optimise.train!), ::Function, ::Tracker.Params, ::Base.Generator{Base.Iterators.PartitionIterator{Array{Int64,1}},getfield(Main, Symbol("##33#34"))}, ::ADAM) at ./none:0