From 0772f807af6ea9d79fb89c78eb5dddd1f8f87d87 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 23 Jan 2025 19:34:46 +0000 Subject: [PATCH] Skipping some tests that are currently incompatible with MMTk --- base/Base.jl | 2 ++ base/gcutils.jl | 11 +++++++ base/timing.jl | 6 ++-- contrib/refresh_checksums.mk | 2 +- src/gc-mmtk.c | 54 +++++++++++++++++++++++++-------- stdlib/Profile/test/allocs.jl | 13 ++++++-- stdlib/Profile/test/runtests.jl | 3 ++ test/checked.jl | 8 +++-- test/cmdlineargs.jl | 49 ++++++++++++++++++------------ test/gc.jl | 12 +++++--- test/misc.jl | 4 ++- 11 files changed, 118 insertions(+), 46 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index 04f732a4309c9..379347f027f69 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -128,6 +128,8 @@ include("sysinfo.jl") include("libc.jl") using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp +const USING_STOCK_GC = occursin("stock", GC.gc_active_impl()) + # These used to be in build_h.jl and are retained for backwards compatibility. # NOTE: keep in sync with `libblastrampoline_jll.libblastrampoline`. const libblas_name = "libblastrampoline" * (Sys.iswindows() ? "-5" : "") diff --git a/base/gcutils.jl b/base/gcutils.jl index 60b8ecdd17d65..d5e6f4597739f 100644 --- a/base/gcutils.jl +++ b/base/gcutils.jl @@ -281,4 +281,15 @@ function logging_enabled() ccall(:jl_is_gc_logging_enabled, Cint, ()) != 0 end +""" + GC.gc_active_impl() + +Return a string stating which GC implementation is being used and possibly +its version according to the list of supported GCs +""" +function gc_active_impl() + unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ())) +end + + end # module GC diff --git a/base/timing.jl b/base/timing.jl index 65c2a643d6a52..61fa73f2eff62 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -109,10 +109,8 @@ function gc_page_utilization_data() return Base.unsafe_wrap(Array, page_utilization_raw, JL_GC_N_MAX_POOLS, own=false) end - -const USING_STOCK_GC = occursin("stock", unsafe_string(ccall(:jl_gc_active_impl, Ptr{UInt8}, ()))) # Full sweep reasons are currently only available for the stock GC -@static if USING_STOCK_GC +@static if Base.USING_STOCK_GC # must be kept in sync with `src/gc-stock.h`` const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP, :FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE] @@ -135,7 +133,7 @@ function full_sweep_reasons() d = Dict{Symbol, Int64}() # populate the dictionary according to the reasons above for the stock GC # otherwise return an empty dictionary for now - @static if USING_STOCK_GC + @static if Base.USING_STOCK_GC reason = cglobal(:jl_full_sweep_reasons, UInt64) reasons_as_array = Base.unsafe_wrap(Vector{UInt64}, reason, length(FULL_SWEEP_REASONS), own=false) for (i, r) in enumerate(FULL_SWEEP_REASONS) diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index 5a787b0b67cb1..fa7cddc705958 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient +BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 5ec1e34cc1acd..2f261a2e8e2fd 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -64,11 +64,37 @@ void jl_gc_init(void) { arraylist_new(&to_finalize, 0); arraylist_new(&finalizer_list_marked, 0); - + gc_num.interval = default_collect_interval; gc_num.allocd = 0; gc_num.max_pause = 0; gc_num.max_memory = 0; + // Necessary if we want to use Julia heap resizing heuristics + uint64_t mem_reserve = 250*1024*1024; // LLVM + other libraries need some amount of memory + uint64_t min_heap_size_hint = mem_reserve + 1*1024*1024; + uint64_t hint = jl_options.heap_size_hint; + + // check if heap size specified on command line + if (jl_options.heap_size_hint == 0) { + char *cp = getenv(HEAP_SIZE_HINT); + if (cp) + hint = parse_heap_size_hint(cp, "JULIA_HEAP_SIZE_HINT=\"[]\""); + } +#ifdef _P64 + if (hint == 0) { + uint64_t constrained_mem = uv_get_constrained_memory(); + if (constrained_mem > 0 && constrained_mem < uv_get_total_memory()) + hint = constrained_mem; + } +#endif + if (hint) { + if (hint < min_heap_size_hint) + hint = min_heap_size_hint; + jl_gc_set_max_memory(hint - mem_reserve); + } + + // MMTK supports setting the heap size using the + // MMTK_MIN_HSIZE and MMTK_MAX_HSIZE environment variables long long min_heap_size; long long max_heap_size; char* min_size_def = getenv("MMTK_MIN_HSIZE"); @@ -77,7 +103,8 @@ void jl_gc_init(void) { char* max_size_def = getenv("MMTK_MAX_HSIZE"); char* max_size_gb = getenv("MMTK_MAX_HSIZE_G"); - // default min heap currently set as Julia's default_collect_interval + // If min and max values are not specified, set them to 0 here + // and use stock heuristics as defined in the binding if (min_size_def != NULL) { char *p; double min_size = strtod(min_size_def, &p); @@ -87,10 +114,9 @@ void jl_gc_init(void) { double min_size = strtod(min_size_gb, &p); min_heap_size = (long) 1024 * 1024 * 1024 * min_size; } else { - min_heap_size = default_collect_interval; + min_heap_size = 0; } - // default max heap currently set as 70% the free memory in the system if (max_size_def != NULL) { char *p; double max_size = strtod(max_size_def, &p); @@ -100,7 +126,7 @@ void jl_gc_init(void) { double max_size = strtod(max_size_gb, &p); max_heap_size = (long) 1024 * 1024 * 1024 * max_size; } else { - max_heap_size = uv_get_free_memory() * 70 / 100; + max_heap_size = 0; } // Assert that the number of stock GC threads is 0; MMTK uses the number of threads in jl_options.ngcthreads @@ -159,7 +185,17 @@ void jl_free_thread_gc_state(struct _jl_tls_states_t *ptls) { } JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) { - // MMTk currently does not allow setting the heap size at runtime +#ifdef _P32 + max_mem = max_mem < MAX32HEAP ? max_mem : MAX32HEAP; +#endif + max_total_memory = max_mem; +} + +JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void) +{ + // FIXME: We should return the max heap size set in MMTk + // when not using Julia's heap resizing heuristics + return max_total_memory; } STATIC_INLINE void maybe_collect(jl_ptls_t ptls) @@ -415,12 +451,6 @@ JL_DLLEXPORT void jl_gc_get_total_bytes(int64_t *bytes) JL_NOTSAFEPOINT *bytes = (num.total_allocd + num.deferred_alloc + num.allocd); } -JL_DLLEXPORT uint64_t jl_gc_get_max_memory(void) -{ - // FIXME: should probably return MMTk's heap size - return max_total_memory; -} - // These are needed to collect MMTk statistics from a Julia program using ccall JL_DLLEXPORT void (jl_mmtk_harness_begin)(void) { diff --git a/stdlib/Profile/test/allocs.jl b/stdlib/Profile/test/allocs.jl index d4930a2b7f5ed..5607783c782f9 100644 --- a/stdlib/Profile/test/allocs.jl +++ b/stdlib/Profile/test/allocs.jl @@ -73,8 +73,14 @@ end @test length(first_alloc.stacktrace) > 0 @test length(string(first_alloc.type)) > 0 - @testset for type in (Task, Vector{Float64},) - @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS + # Issue #57103: This test does not work with MMTk because of fastpath + # allocation which never calls the allocation profiler. + # TODO: We should port these observability tools (e.g. allocation + # profiler and heap snapshot) to MMTk + @static if Base.USING_STOCK_GC + @testset for type in (Task, Vector{Float64},) + @test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS + end end # TODO: it would be nice to assert that these tasks @@ -143,6 +149,8 @@ end @test length([a for a in prof.allocs if a.type == String]) >= 1 end +# FIXME: Issue #57103 disabling test for MMTk. +@static if Base.USING_STOCK_GC @testset "alloc profiler catches allocs from codegen" begin @eval begin struct MyType x::Int; y::Int end @@ -162,6 +170,7 @@ end @test length(prof.allocs) >= 1 @test length([a for a in prof.allocs if a.type == MyType]) >= 1 end +end @testset "alloc profiler catches allocs from buffer resize" begin f(a) = for _ in 1:100; push!(a, 1); end diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index b73a2a618011b..a5a604d7923d0 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -344,6 +344,8 @@ end @test only(node.down).first == lidict[8] end +# FIXME: Issue #57103: heap snapshots is not currently supported in MMTk +@static if Base.USING_STOCK_GC @testset "HeapSnapshot" begin tmpdir = mktempdir() @@ -374,6 +376,7 @@ end rm(fname) rm(tmpdir, force = true, recursive = true) end +end @testset "PageProfile" begin fname = "$(getpid())_$(time_ns())" diff --git a/test/checked.jl b/test/checked.jl index 4031918a38730..b93c8796162c5 100644 --- a/test/checked.jl +++ b/test/checked.jl @@ -331,8 +331,12 @@ end @test checked_pow(BigInt(2), 2) == BigInt(4) @test checked_pow(BigInt(2), 100) == BigInt(1267650600228229401496703205376) - # Perf test: Make sure BigInts allocs don't scale with the power: - @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9 + # FIXME: Issue #57103: the following test may fail because + # allocation may not be logged via MMTk's fastpath allocation + @static if Base.USING_STOCK_GC + # Perf test: Make sure BigInts allocs don't scale with the power: + @test @allocations(checked_pow(BigInt(2), 2)) ≈ @allocations(checked_pow(BigInt(2), 10000)) rtol=0.9 + end end @testset "Additional tests" begin diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 3ff7836223b84..ec7a51e804d3f 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -383,29 +383,33 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test p.exitcode == 1 && p.termsignal == 0 end - # --gcthreads - code = "print(Threads.ngcthreads())" - cpu_threads = ccall(:jl_effective_threads, Int32, ()) - @test string(cpu_threads) == - read(`$exename --threads auto -e $code`, String) == - read(`$exename --threads=auto -e $code`, String) == - read(`$exename -tauto -e $code`, String) == - read(`$exename -t auto -e $code`, String) - for nt in (nothing, "1") - withenv("JULIA_NUM_GC_THREADS" => nt) do - @test read(`$exename --gcthreads=2 -e $code`, String) == "2" - end - withenv("JULIA_NUM_GC_THREADS" => nt) do - @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3" + # FIXME: Issue #57103 --gcthreads does not have the same semantics + # for Stock GC and MMTk, so the tests below are specific to the Stock GC + @static if Base.USING_STOCK_GC + # --gcthreads + code = "print(Threads.ngcthreads())" + cpu_threads = ccall(:jl_effective_threads, Int32, ()) + @test string(cpu_threads) == + read(`$exename --threads auto -e $code`, String) == + read(`$exename --threads=auto -e $code`, String) == + read(`$exename -tauto -e $code`, String) == + read(`$exename -t auto -e $code`, String) + for nt in (nothing, "1") + withenv("JULIA_NUM_GC_THREADS" => nt) do + @test read(`$exename --gcthreads=2 -e $code`, String) == "2" + end + withenv("JULIA_NUM_GC_THREADS" => nt) do + @test read(`$exename --gcthreads=2,1 -e $code`, String) == "3" + end end - end - withenv("JULIA_NUM_GC_THREADS" => 2) do - @test read(`$exename -e $code`, String) == "2" - end + withenv("JULIA_NUM_GC_THREADS" => 2) do + @test read(`$exename -e $code`, String) == "2" + end - withenv("JULIA_NUM_GC_THREADS" => "2,1") do - @test read(`$exename -e $code`, String) == "3" + withenv("JULIA_NUM_GC_THREADS" => "2,1") do + @test read(`$exename -e $code`, String) == "3" + end end # --machine-file @@ -1182,6 +1186,10 @@ end end end +# FIXME: Issue #57103: MMTK currently does not use --heap-size-hint since it only +# supports setting up a hard limit unlike the Stock GC +# which takes it as a soft limit. For now, we skip the tests below for MMTk +@static if Base.USING_STOCK_GC @testset "heap size hint" begin #heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.) @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)" @@ -1201,6 +1209,7 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end +end ## `Main.main` entrypoint diff --git a/test/gc.jl b/test/gc.jl index c532f17f04eb5..3e9f03ef40d92 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -67,6 +67,9 @@ end run_gctest("gc/chunks.jl") end +#FIXME: Issue #57103 disabling tests for MMTk, since +# they rely on information that is specific to the stock GC. +@static if Base.USING_STOCK_GC @testset "GC page metrics" begin run_nonzero_page_utilization_test() run_pg_size_test() @@ -76,13 +79,14 @@ end issue_54275_test() end -@testset "Base.GC docstrings" begin - @test isempty(Docs.undocumented_names(GC)) -end - @testset "Full GC reasons" begin full_sweep_reasons_test() end +end + +@testset "Base.GC docstrings" begin + @test isempty(Docs.undocumented_names(GC)) +end #testset doesn't work here because this needs to run in top level #Check that we ensure objects in toplevel exprs are rooted diff --git a/test/misc.jl b/test/misc.jl index 7070fd49f5f36..070952db89032 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1453,7 +1453,8 @@ end @test_throws ErrorException finalizer(x->nothing, 1) @test_throws ErrorException finalizer(C_NULL, 1) - +# FIXME: Issue #57103 Test is specific to Stock GC +@static if Base.USING_STOCK_GC @testset "GC utilities" begin GC.gc() GC.gc(true); GC.gc(false) @@ -1473,6 +1474,7 @@ end @test occursin("GC: pause", read(tmppath, String)) end end +end @testset "fieldtypes Module" begin @test fieldtypes(Module) === ()