Wma/shard levels

[willow-ahrens]

Add ShardLevels and rework parallelism in Finch

[github-actions]

Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit

JuliaFormatter

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 142 in 2289266

VirtualShardLevel(device_2, lvl_2, sym, ptr, task, val, typeof(level_fill_value(Lvl)), Device, Lvl, Ptr, Task, Val)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 147 in 2289266

virtual_level_resize!(ctx, lvl::VirtualShardLevel, dims...) = (lvl.lvl = virtual_level_resize!(ctx, lvl.lvl, dims...); lvl)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 154 in 2289266

push_preamble!(ctx, quote

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Lines 157 to 158 in 2289266

	end)
	push_epilogue!(ctx, quote

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 160 in 2289266

end)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 165 in 2289266

push_preamble!(ctx,

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 167 in 2289266

lvl_2 = virtualize(ctx_2, :($(lvl.ex).val[$(ctx_2(get_task_num(get_task(ctx_2))))]), lvl.Lvl) #TODO should this virtualize the eltype of Val?

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 169 in 2289266

end

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Lines 191 to 195 in 2289266

	push_preamble!(ctx, quote
	Finch.resize_if_smaller!($(lvl.task), $(ctx(pos_stop)))
	Finch.resize_if_smaller!($(lvl.ptr), $(ctx(pos_stop)))
	Finch.fill_range!($(lvl.task), $(ctx(pos_start)), $(ctx(pos_stop)), 0)
	end)

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Lines 220 to 221 in 2289266

	return Thunk(
	body = (ctx) -> begin

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Lines 231 to 232 in 2289266

	return Thunk(
	body = (ctx) -> begin

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 241 in 2289266

end

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 244 in 2289266

end

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 271 in 2289266

return Thunk(

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 279 in 2289266

$(contain(ctx_2->assemble_level!(ctx_2, lvl.lvl, value(qos_fill, Tp), value(qos_stop, Tp)), ctx))

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 288 in 2289266

body = (ctx) -> VirtualHollowSubFiber(lvl.lvl, value(qos), dirty),

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 295 in 2289266

end

---

[JuliaFormatter] _{reported by reviewdog 🐶}

Finch.jl/src/tensors/levels/shard_levels.jl

Line 297 in 2289266

end

[codecov]

Codecov Report

Attention: Patch coverage is 25.18892% with 297 lines in your changes missing coverage. Please review.

Files with missing lines	Patch %	Lines
src/tensors/levels/shard_levels.jl	0.00%	259 Missing ⚠️
src/architecture.jl	0.00%	16 Missing ⚠️
src/tensors/levels/sparse_rle_levels.jl	52.00%	12 Missing ⚠️
ext/SparseArraysExt.jl	57.89%	8 Missing ⚠️
src/tensors/levels/dense_levels.jl	50.00%	2 Missing ⚠️

Files with missing lines	Coverage Δ
src/Finch.jl	94.68% <ø> (ø)
src/tensors/levels/dense_rle_levels.jl	79.55% <100.00%> (-4.00%)	⬇️
src/tensors/levels/separate_levels.jl	69.93% <ø> (-4.20%)	⬇️
src/tensors/levels/sparse_band_levels.jl	75.40% <100.00%> (ø)
src/tensors/levels/sparse_bytemap_levels.jl	79.44% <100.00%> (-4.75%)	⬇️
src/tensors/levels/sparse_coo_levels.jl	83.40% <100.00%> (-4.81%)	⬇️
src/tensors/levels/sparse_dict_levels.jl	79.48% <100.00%> (-3.42%)	⬇️
src/tensors/levels/sparse_interval_levels.jl	70.23% <100.00%> (-7.74%)	⬇️
src/tensors/levels/sparse_list_levels.jl	89.09% <100.00%> (-3.80%)	⬇️
src/tensors/levels/sparse_vbl_levels.jl	77.02% <100.00%> (ø)
... and 5 more

... and 27 files with indirect coverage changes

🚀 New features to boost your workflow:

• ❄️ Test Analytics: Detect flaky tests, report on failures, and find test suite problems.

[Paramuths]

Why do we call transfer on lvl twice?

[Paramuths]

Do we need device argument?