From 099d07fcb685e4d3e94de60223989039637b059a Mon Sep 17 00:00:00 2001 From: Neven Sajko Date: Sun, 29 Sep 2024 11:41:59 +0200 Subject: [PATCH 01/26] fix rawbigints OOB issues (#55917) Fixes issues introduced in #50691 and found in #55906: * use `@inbounds` and `@boundscheck` macros in rawbigints, for catching OOB with `--check-bounds=yes` * fix OOB in `truncate` (cherry picked from commit 17445fe752b7b99633ca306af0981baca9f66bda) --- test/mpfr.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/mpfr.jl b/test/mpfr.jl index 63365cf803deb..18207adc5a5e9 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -1055,3 +1055,12 @@ end end end end + +@testset "RawBigInt truncation OOB read" begin + @testset "T: $T" for T ∈ (UInt8, UInt16, UInt32, UInt64, UInt128) + v = Base.RawBigInt{T}("a"^sizeof(T), 1) + @testset "bit_count: $bit_count" for bit_count ∈ (0:10:80) + @test Base.truncated(UInt128, v, bit_count) isa Any + end + end +end From 359b9cc4a7556cf0b0a69b4aca3994a7cb940934 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 1 Oct 2024 10:32:57 -0400 Subject: [PATCH 02/26] Fix logic in `?` docstring example (#55945) (cherry picked from commit 81ce6a41d737f15d8bbc2788190dcb5565e20b8b) --- base/docs/basedocs.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 3b666662c1962..0c07de336248b 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -931,11 +931,14 @@ expression, rather than the side effects that evaluating `b` or `c` may have. See the manual section on [control flow](@ref man-conditional-evaluation) for more details. # Examples -``` +```jldoctest julia> x = 1; y = 2; -julia> x > y ? println("x is larger") : println("y is larger") -y is larger +julia> x > y ? println("x is larger") : println("x is not larger") +x is not larger + +julia> x > y ? "x is larger" : x == y ? "x and y are equal" : "y is larger" +"y is larger" ``` """ kw"?", kw"?:" From 4eae986770b152264f6f087a1bc6008ba3cdb08b Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 2 Oct 2024 07:27:35 -0400 Subject: [PATCH 03/26] REPL: make UndefVarError aware of imported modules (#55932) (cherry picked from commit fbb3e1175d52abec0ff4ca83d8c9e126d9f8a06b) --- base/experimental.jl | 4 ++-- stdlib/REPL/src/REPL.jl | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/base/experimental.jl b/base/experimental.jl index 58c7258120f3f..f0cb510dcc651 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -319,9 +319,9 @@ function show_error_hints(io, ex, args...) for handler in hinters try @invokelatest handler(io, ex, args...) - catch err + catch tn = typeof(handler).name - @error "Hint-handler $handler for $(typeof(ex)) in $(tn.module) caused an error" + @error "Hint-handler $handler for $(typeof(ex)) in $(tn.module) caused an error" exception=current_exceptions() end end end diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f47772f7b9ec0..0d87096b81e21 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -81,7 +81,17 @@ end function _UndefVarError_warnfor(io::IO, m::Module, var::Symbol) Base.isbindingresolved(m, var) || return false (Base.isexported(m, var) || Base.ispublic(m, var)) || return false - print(io, "\nHint: a global variable of this name also exists in $m.") + active_mod = Base.active_module() + print(io, "\nHint: ") + if isdefined(active_mod, Symbol(m)) + print(io, "a global variable of this name also exists in $m.") + else + if Symbol(m) == var + print(io, "$m is loaded but not imported in the active module $active_mod.") + else + print(io, "a global variable of this name may be made accessible by importing $m in the current active module $active_mod") + end + end return true end From f8a5b981921fa91b8960fd06de4664dca068b381 Mon Sep 17 00:00:00 2001 From: Michael Cho Date: Wed, 2 Oct 2024 20:46:36 -0400 Subject: [PATCH 04/26] [build] avoid libedit linkage and align libccalllazy* SONAMEs (#55968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While building the 1.11.0-rc4 in Homebrew[^1] in preparation for 1.11.0 release (and to confirm Sequoia successfully builds) I noticed some odd linkage for our Linux builds, which included of: 1. LLVM libraries were linking to `libedit.so`, e.g. ``` Dynamic Section: NEEDED libedit.so.0 NEEDED libz.so.1 NEEDED libzstd.so.1 NEEDED libstdc++.so.6 NEEDED libm.so.6 NEEDED libgcc_s.so.1 NEEDED libc.so.6 NEEDED ld-linux-x86-64.so.2 SONAME libLLVM-16jl.so ``` CMakeCache.txt showed ``` //Use libedit if available. LLVM_ENABLE_LIBEDIT:BOOL=ON ``` Which might be overriding `HAVE_LIBEDIT` at https://github.com/JuliaLang/llvm-project/blob/julia-release/16.x/llvm/cmake/config-ix.cmake#L222-L225. So just added `LLVM_ENABLE_LIBEDIT` 2. Wasn't sure if there was a reason for this but `libccalllazy*` had mismatched SONAME: ```console ❯ objdump -p lib/julia/libccalllazy* | rg '\.so' lib/julia/libccalllazybar.so: file format elf64-x86-64 NEEDED ccalllazyfoo.so SONAME ccalllazybar.so lib/julia/libccalllazyfoo.so: file format elf64-x86-64 SONAME ccalllazyfoo.so ``` Modifying this, but can drop if intentional. --- [^1]: https://github.com/Homebrew/homebrew-core/pull/192116 (cherry picked from commit 77c5875b3cbe85e7fb0bb5a7e796809c901ede95) --- deps/llvm.mk | 2 +- src/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 36d6e8547783d..97a275b1af4bf 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -100,7 +100,7 @@ endif LLVM_CMAKE += -DLLVM_TOOLS_INSTALL_DIR=$(call rel_path,$(build_prefix),$(build_depsbindir)) LLVM_CMAKE += -DLLVM_UTILS_INSTALL_DIR=$(call rel_path,$(build_prefix),$(build_depsbindir)) LLVM_CMAKE += -DLLVM_INCLUDE_UTILS=ON -DLLVM_INSTALL_UTILS=ON -LLVM_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERMINFO=Off -DHAVE_LIBEDIT=Off +LLVM_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERMINFO=Off -DHAVE_LIBEDIT=Off -DLLVM_ENABLE_LIBEDIT=OFF ifeq ($(LLVM_ASSERTIONS), 1) LLVM_CMAKE += -DLLVM_ENABLE_ASSERTIONS:BOOL=ON endif # LLVM_ASSERTIONS diff --git a/src/Makefile b/src/Makefile index 9c48c04b25f9e..bf9001e5fba93 100644 --- a/src/Makefile +++ b/src/Makefile @@ -286,10 +286,10 @@ endif $(INSTALL_NAME_CMD)libccalltest.$(SHLIB_EXT) $@ $(build_shlibdir)/libccalllazyfoo.$(SHLIB_EXT): $(SRCDIR)/ccalllazyfoo.c - @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,ccalllazyfoo.$(SHLIB_EXT))) + @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,libccalllazyfoo.$(SHLIB_EXT))) $(build_shlibdir)/libccalllazybar.$(SHLIB_EXT): $(SRCDIR)/ccalllazybar.c $(build_shlibdir)/libccalllazyfoo.$(SHLIB_EXT) - @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,ccalllazybar.$(SHLIB_EXT)) -lccalllazyfoo) + @$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,libccalllazybar.$(SHLIB_EXT)) -lccalllazyfoo) $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvmcalltest.cpp $(LLVM_CONFIG_ABSOLUTE) @$(call PRINT_CC, $(CXX) $(LLVM_CXXFLAGS) $(FLAGS) $(CPPFLAGS) $(CXXFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(NO_WHOLE_ARCHIVE) $(CG_LLVMLINK)) -lpthread From 4f81618a12385e562a32a66e4caa2526d8642f6d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 3 Oct 2024 16:23:21 -0400 Subject: [PATCH 05/26] fix comma logic in time_print (#55977) Minor formatting fix (cherry picked from commit 42737f79e02bbaf444a9d93e6668b3c55cdb8a6e) --- base/timing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/timing.jl b/base/timing.jl index ecb67a2375d92..ac2f3ab29f1b7 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -180,7 +180,7 @@ function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, lock_confl print(io, length(timestr) < 10 ? (" "^(10 - length(timestr))) : "") end print(io, timestr, " seconds") - parens = bytes != 0 || allocs != 0 || gctime > 0 || compile_time > 0 + parens = bytes != 0 || allocs != 0 || gctime > 0 || compile_time > 0 || lock_conflicts > 0 parens && print(io, " (") if bytes != 0 || allocs != 0 allocs, ma = prettyprint_getunits(allocs, length(_cnt_units), Int64(1000)) @@ -202,7 +202,7 @@ function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, lock_confl print(io, ", ", lock_conflicts, " lock conflict$plural") end if compile_time > 0 - if bytes != 0 || allocs != 0 || gctime > 0 + if bytes != 0 || allocs != 0 || gctime > 0 || lock_conflicts > 0 print(io, ", ") end print(io, Ryu.writefixed(Float64(100*compile_time/elapsedtime), 2), "% compilation time") From 6a00b178fc121b5d3e9225fb8eb90e66ff458e15 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Oct 2024 09:57:53 -0400 Subject: [PATCH 06/26] `@time` actually fix time report commas & add tests (#55982) https://github.com/JuliaLang/julia/pull/55977 looked simple but wasn't quite right because of a bad pattern in the lock conflicts report section. So fix and add tests. (cherry picked from commit 636a35d83ca16d2077fc507701f41d50f409c7a5) --- base/timing.jl | 7 +++++-- test/misc.jl | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/base/timing.jl b/base/timing.jl index ac2f3ab29f1b7..dcdd052926b67 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -180,7 +180,7 @@ function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, lock_confl print(io, length(timestr) < 10 ? (" "^(10 - length(timestr))) : "") end print(io, timestr, " seconds") - parens = bytes != 0 || allocs != 0 || gctime > 0 || compile_time > 0 || lock_conflicts > 0 + parens = bytes != 0 || allocs != 0 || gctime > 0 || lock_conflicts > 0 || compile_time > 0 parens && print(io, " (") if bytes != 0 || allocs != 0 allocs, ma = prettyprint_getunits(allocs, length(_cnt_units), Int64(1000)) @@ -198,8 +198,11 @@ function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, lock_confl print(io, Ryu.writefixed(Float64(100*gctime/elapsedtime), 2), "% gc time") end if lock_conflicts > 0 + if bytes != 0 || allocs != 0 || gctime > 0 + print(io, ", ") + end plural = lock_conflicts == 1 ? "" : "s" - print(io, ", ", lock_conflicts, " lock conflict$plural") + print(io, lock_conflicts, " lock conflict$plural") end if compile_time > 0 if bytes != 0 || allocs != 0 || gctime > 0 || lock_conflicts > 0 diff --git a/test/misc.jl b/test/misc.jl index a257b0e79a4ac..0afbd2b89578a 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -339,6 +339,15 @@ let foo() = "message" @test @timev foo() true end +# this is internal, but used for easy testing +@test sprint(Base.time_print, 1e9) == " 1.000000 seconds" +@test sprint(Base.time_print, 1e9, 111, 0, 222) == " 1.000000 seconds (222 allocations: 111 bytes)" +@test sprint(Base.time_print, 1e9, 111, 0.5e9, 222) == " 1.000000 seconds (222 allocations: 111 bytes, 50.00% gc time)" +@test sprint(Base.time_print, 1e9, 111, 0, 222, 333) == " 1.000000 seconds (222 allocations: 111 bytes, 333 lock conflicts)" +@test sprint(Base.time_print, 1e9, 0, 0, 0, 333) == " 1.000000 seconds (333 lock conflicts)" +@test sprint(Base.time_print, 1e9, 111, 0, 222, 333, 0.25e9) == " 1.000000 seconds (222 allocations: 111 bytes, 333 lock conflicts, 25.00% compilation time)" +@test sprint(Base.time_print, 1e9, 111, 0.5e9, 222, 333, 0.25e9, 0.175e9) == " 1.000000 seconds (222 allocations: 111 bytes, 50.00% gc time, 333 lock conflicts, 25.00% compilation time: 70% of which was recompilation)" + # @showtime @test @showtime true let foo() = true From 76967d4deb47b584ea9faeb73e4a25dbc0f3ef4e Mon Sep 17 00:00:00 2001 From: Neven Sajko Date: Sat, 5 Oct 2024 05:29:19 +0200 Subject: [PATCH 07/26] Profile: document heap snapshot viewing tools (#55743) (cherry picked from commit 3a132cfc7661f850340d0360df6d3279d1aa7e16) --- stdlib/Profile/docs/src/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stdlib/Profile/docs/src/index.md b/stdlib/Profile/docs/src/index.md index 5b4db77b9cb16..0b358e5decfa9 100644 --- a/stdlib/Profile/docs/src/index.md +++ b/stdlib/Profile/docs/src/index.md @@ -155,3 +155,8 @@ julia> Profile.HeapSnapshot.assemble_snapshot("snapshot", "snapshot.heapsnapshot The resulting heap snapshot file can be uploaded to chrome devtools to be viewed. For more information, see the [chrome devtools docs](https://developer.chrome.com/docs/devtools/memory-problems/heap-snapshots/#view_snapshots). +An alternative for analyzing Chromium heap snapshots is with the VS Code extension +`ms-vscode.vscode-js-profile-flame`. + +The Firefox heap snapshots are of a different format, and Firefox currently may +*not* be used for viewing the heap snapshots generated by Julia. From bc75cd1520866af5774c4edd79fc18e6b1ba72bf Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sat, 5 Oct 2024 01:24:42 -0300 Subject: [PATCH 08/26] [REPL] Fix #55850 by using `safe_realpath` instead of `abspath` in `projname` (#55851) (cherry picked from commit fb77d60fb7088a3b8ccac73ea3476b9a8c88d455) --- stdlib/REPL/src/Pkg_beforeload.jl | 2 +- stdlib/REPL/test/repl.jl | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/stdlib/REPL/src/Pkg_beforeload.jl b/stdlib/REPL/src/Pkg_beforeload.jl index a73282dd6bdd3..c5f7f107fcadc 100644 --- a/stdlib/REPL/src/Pkg_beforeload.jl +++ b/stdlib/REPL/src/Pkg_beforeload.jl @@ -79,7 +79,7 @@ function projname(project_file::String) end for depot in Base.DEPOT_PATH envdir = joinpath(depot, "environments") - if startswith(abspath(project_file), abspath(envdir)) + if startswith(safe_realpath(project_file), safe_realpath(envdir)) return "@" * name end end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 671c45a6bf2b1..b5007c64c72aa 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1781,11 +1781,20 @@ end @testset "Dummy Pkg prompt" begin # do this in an empty depot to test default for new users - withenv("JULIA_DEPOT_PATH" => mktempdir(), "JULIA_LOAD_PATH" => nothing) do + withenv("JULIA_DEPOT_PATH" => mktempdir() * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do prompt = readchomp(`$(Base.julia_cmd()[1]) --startup-file=no -e "using REPL; print(REPL.Pkg_promptf())"`) @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> " end + # Issue 55850 + tmp_55850 = mktempdir() + tmp_sym_link = joinpath(tmp_55850, "sym") + symlink(tmp_55850, tmp_sym_link; dir_target=true) + withenv("JULIA_DEPOT_PATH" => tmp_sym_link * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do + prompt = readchomp(`$(Base.julia_cmd()[1]) --startup-file=no -e "using REPL; print(REPL.projname(REPL.find_project_file()))"`) + @test prompt == "@v$(VERSION.major).$(VERSION.minor)" + end + get_prompt(proj::String) = readchomp(`$(Base.julia_cmd()[1]) --startup-file=no $(proj) -e "using REPL; print(REPL.Pkg_promptf())"`) @test get_prompt("--project=$(pkgdir(REPL))") == "(REPL) pkg> " From a1b972f7fdea628138f23ef4010f29b2327ca6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Sat, 5 Oct 2024 08:34:48 +0100 Subject: [PATCH 09/26] Avoid `stat`-ing stdlib path if it's unreadable (#55992) (cherry picked from commit 5d12c6de4399e62e5e863a9b25b676ce4e26ce97) --- base/loading.jl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index c5db12d76fc26..360da566a5f88 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -3778,10 +3778,17 @@ end # now check if this file's content hash has changed relative to its source files if stalecheck - if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) - @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" - record_reason(reasons, "wrong source") - return true # cache file was compiled from a different path + if !samefile(includes[1].filename, modpath) + # In certain cases the path rewritten by `fixup_stdlib_path` may + # point to an unreadable directory, make sure we can `stat` the + # file before comparing it with `modpath`. + stdlib_path = fixup_stdlib_path(includes[1].filename) + if !(isreadable(stdlib_path) && samefile(stdlib_path, modpath)) + !samefile(fixup_stdlib_path(includes[1].filename), modpath) + @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" + record_reason(reasons, "wrong source") + return true # cache file was compiled from a different path + end end for (modkey, req_modkey) in requires # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` From 6b8ef313419cbfbe8d5554e85e1169df7a3158ce Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 7 Oct 2024 02:31:59 -0400 Subject: [PATCH 10/26] Fix no-arg `ScopedValues.@with` within a scope (#56019) Fixes https://github.com/JuliaLang/julia/issues/56017 (cherry picked from commit 57e3c9e4bfd2bc3d54a4923066ed2c3f087b1311) --- base/scopedvalues.jl | 2 ++ test/scopedvalues.jl | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 6ccd4687c5c65..39e3c2c076718 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -85,6 +85,8 @@ struct Scope values::ScopeStorage end +Scope(scope::Scope) = scope + function Scope(parent::Union{Nothing, Scope}, key::ScopedValue{T}, value) where T val = convert(T, value) if parent === nothing diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index 3af6a3f065c8b..f3cd5101d0c03 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -135,6 +135,12 @@ end @test sval[] == 1 @test sval_float[] == 1.0 end + @with sval=>2 sval_float=>2.0 begin + @with begin + @test sval[] == 2 + @test sval_float[] == 2.0 + end + end end @testset "isassigned" begin From d21a6e8438cd6d71c1c01699126e550fa7deb055 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:30:50 -0400 Subject: [PATCH 11/26] =?UTF-8?q?=F0=9F=A4=96=20[backports-release-1.11]?= =?UTF-8?q?=20Bump=20the=20Pkg=20stdlib=20from=206ceafca8e=20to=20aba90d22?= =?UTF-8?q?b=20(#56032)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: release-1.11 Julia branch: backports-release-1.11 Old commit: 6ceafca8e New commit: aba90d22b Julia version: 1.11.0 Pkg version: 1.11.0 Bump invoked by: @IanButterworth Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/6ceafca8e2adab1b248b19e468d432c167dd7b7e...aba90d22b42a993b118276caa8df5146776d3844 ``` $ git log --oneline 6ceafca8e..aba90d22b aba90d22b Merge pull request #4037 from JuliaLang/backports-release-1.11 76eaa4caa Fix julia#55850 by using safe_realpath instead of abspath in projname (#4025) df38587fb warn if General is installed via the old slow methods (#4022) 1475b628a update package extension naming docs (#4000) 72dc85e80 Tweak sentence syntax in getting-started.md (#4020) 0b2397089 make `add` and `dev` on a package remove it from the set of weak dependencies (#3865) ee2d51054 collect e.g. weak deps from project even if it is not a package (#3852) ``` Co-authored-by: Dilum Aluthge --- .../Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 | 1 - .../Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 | 1 - .../Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/md5 | 1 + .../Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/md5 create mode 100644 deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/sha512 diff --git a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 deleted file mode 100644 index 96d256e63fc13..0000000000000 --- a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -83f567b5c65ae36ebd9b5878cf009513 diff --git a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 b/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 deleted file mode 100644 index c4b322eac0395..0000000000000 --- a/deps/checksums/Pkg-6ceafca8e2adab1b248b19e468d432c167dd7b7e.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -bc1e6de2c679b862caeb5c1fb6c3187c9da4dc2e37dc89b5cdf1498c5a26fbe7516773b53c039927abd89a1a9e8215f410b08a3da9d1b6489fe9ca2da60e57d2 diff --git a/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/md5 b/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/md5 new file mode 100644 index 0000000000000..f089de090242f --- /dev/null +++ b/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/md5 @@ -0,0 +1 @@ +d826591b2b1dd656aef3f278045ad8e2 diff --git a/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/sha512 b/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/sha512 new file mode 100644 index 0000000000000..6fe05afe2c667 --- /dev/null +++ b/deps/checksums/Pkg-aba90d22b42a993b118276caa8df5146776d3844.tar.gz/sha512 @@ -0,0 +1 @@ +191c44b8a3520d727c2340ac4d0e43984cd0be89fa2abe8f7f8e0c161c434037ff3f974977621e375b59bc4cdd2489d9994fa26e9bc5543154820b1d64605a17 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 90a7c19aae59c..56fe7ac37a8f2 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.11 -PKG_SHA1 = 6ceafca8e2adab1b248b19e468d432c167dd7b7e +PKG_SHA1 = aba90d22b42a993b118276caa8df5146776d3844 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 794ec9d0695f80f562263445cea1c53b248d5cb8 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 8 Oct 2024 13:03:32 -0400 Subject: [PATCH 12/26] Don't show keymap `@error` for hints (#56041) It's too disruptive to show errors for hints. The error will still be shown if tab is pressed. Helps issues like https://github.com/JuliaLang/julia/issues/56037 (cherry picked from commit 8d515ed8ad443d4bb8c530468d5b7ae0f3b50a0b) --- stdlib/REPL/src/LineEdit.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 431e78551ed2f..76e6ee42fd53b 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -382,7 +382,13 @@ function check_for_hint(s::MIState) # Requires making space for them earlier in refresh_multi_line return clear_hint(st) end - completions, partial, should_complete = complete_line(st.p.complete, st, s.active_module; hint = true)::Tuple{Vector{String},String,Bool} + + completions, partial, should_complete = try + complete_line(st.p.complete, st, s.active_module; hint = true)::Tuple{Vector{String},String,Bool} + catch + @debug "error completing line for hint" exception=current_exceptions() + return clear_hint(st) + end isempty(completions) && return clear_hint(st) # Don't complete for single chars, given e.g. `x` completes to `xor` if length(partial) > 1 && should_complete From a9fd6db07e60e94498ae7f4685bb28d2b838aa7c Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 9 Oct 2024 06:03:41 -0400 Subject: [PATCH 13/26] effects: fix `Base.@_noub_meta` (#56061) This had the incorrect number of arguments to `Expr(:purity, ...)` causing it to be silently ignored. --- base/essentials.jl | 3 ++- src/method.c | 2 ++ test/compiler/effects.jl | 7 ++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index badb7b0853b3b..33c593726f0fc 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -348,7 +348,8 @@ macro _noub_meta() #=:inaccessiblememonly=#false, #=:noub=#true, #=:noub_if_noinbounds=#false, - #=:consistent_overlay=#false)) + #=:consistent_overlay=#false, + #=:nortcall=#false)) end # can be used in place of `@assume_effects :notaskstate` (supposed to be used for bootstrapping) macro _notaskstate_meta() diff --git a/src/method.c b/src/method.c index 7a5d2af49cffd..1d4085bd5e71d 100644 --- a/src/method.c +++ b/src/method.c @@ -370,6 +370,8 @@ static void jl_code_info_set_ir(jl_code_info_t *li, jl_expr_t *ir) if (consistent_overlay) li->purity.overrides.ipo_consistent_overlay = consistent_overlay; int8_t nortcall = jl_unbox_bool(jl_exprarg(ma, 10)); if (nortcall) li->purity.overrides.ipo_nortcall = nortcall; + } else { + assert(jl_expr_nargs(ma) == 0); } } else diff --git a/test/compiler/effects.jl b/test/compiler/effects.jl index b48d00b048d42..98681ca6ce997 100644 --- a/test/compiler/effects.jl +++ b/test/compiler/effects.jl @@ -858,7 +858,12 @@ end # @test !Core.Compiler.is_nothrow(effects) # end #end -# + +@test Core.Compiler.is_noub(Base.infer_effects(Base._growbeg!, (Vector{Int}, Int))) +@test Core.Compiler.is_noub(Base.infer_effects(Base._growbeg!, (Vector{Any}, Int))) +@test Core.Compiler.is_noub(Base.infer_effects(Base._growend!, (Vector{Int}, Int))) +@test Core.Compiler.is_noub(Base.infer_effects(Base._growend!, (Vector{Any}, Int))) + # tuple indexing # -------------- From 1d082076a97cffbae9f3de7e6d0c3cde3659806f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 7 Oct 2024 16:01:23 +0200 Subject: [PATCH 14/26] Sockets: Warn when local network access not granted. (#56023) Works around https://github.com/JuliaLang/julia/issues/56022 (cherry picked from commit c7071e1eb2369211cf02a1e7dbae365f5fba3fc9) --- stdlib/Sockets/test/runtests.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/Sockets/test/runtests.jl b/stdlib/Sockets/test/runtests.jl index 2a812388d8373..cec1c2540d048 100644 --- a/stdlib/Sockets/test/runtests.jl +++ b/stdlib/Sockets/test/runtests.jl @@ -452,6 +452,8 @@ end catch e if isa(e, Base.IOError) && Base.uverrorname(e.code) == "EPERM" @warn "UDP IPv4 broadcast test skipped (permission denied upon send, restrictive firewall?)" + elseif Sys.isapple() && isa(e, Base.IOError) && Base.uverrorname(e.code) == "EHOSTUNREACH" + @warn "UDP IPv4 broadcast test skipped (local network access not granded?)" else rethrow() end From 3c9cf36081829d5e5b96b669a6154f9db2363933 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 10 Apr 2024 20:08:10 +0200 Subject: [PATCH 15/26] allow extensions to trigger from packages in [deps] (#54009) There is a use case where you have a weak dependency (for one of your extensions) that is misbehaving and you quickly want to try debug that issue. A workflow that feels reasonable for this could be: ``` pkg> dev WeakDependency julia> using Package, WeakDependency ``` This doesn't work right now for two reasons: 1. Doing the `dev WeakDependency` will add the dependency to `[deps]` but not remove it from `[weakdeps]` which means you all of a sudden are in the scenario described in https://pkgdocs.julialang.org/v1/creating-packages/#Transition-from-normal-dependency-to-extension which is not what is desired. 2. The extension will not actually load because you can right now only trigger extensions from weak deps getting loaded, not from deps getting loaded. Point 1. is fixed by https://github.com/JuliaLang/Pkg.jl/pull/3865 Point 2. is fixed by this PR. (cherry picked from commit f46cb4c67cd2ce7f88da778f5c0355c1100b5ccd) --- base/loading.jl | 53 +++++++++++-------- base/precompilation.jl | 2 +- doc/src/manual/code-loading.md | 12 ++--- test/loading.jl | 5 ++ .../EnvWithHasExtensions/Manifest.toml | 6 ++- .../Extensions/ExtDep3.jl/Project.toml | 4 ++ .../Extensions/ExtDep3.jl/src/ExtDep3.jl | 5 ++ .../HasDepWithExtensions.jl/Manifest.toml | 11 +++- .../HasDepWithExtensions.jl/Project.toml | 1 + .../Extensions/HasExtensions.jl/Manifest.toml | 9 ++-- .../Extensions/HasExtensions.jl/Project.toml | 4 ++ .../HasExtensions.jl/ext/ExtensionDep.jl | 9 ++++ .../HasExtensions.jl/src/HasExtensions.jl | 1 + 13 files changed, 85 insertions(+), 37 deletions(-) create mode 100644 test/project/Extensions/ExtDep3.jl/Project.toml create mode 100644 test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl create mode 100644 test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl diff --git a/base/loading.jl b/base/loading.jl index 360da566a5f88..140131b26e7b2 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1405,13 +1405,12 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi proj_pkg = project_file_name_uuid(implicit_project_file, pkg.name) if pkg == proj_pkg d_proj = parsed_toml(implicit_project_file) - weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) - end + weakdeps = get(Dict{String, Any}, d_proj, "weakdeps")::Dict{String,Any} + deps = get(Dict{String, Any}, d_proj, "deps")::Dict{String,Any} + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end # Now look in manifest @@ -1426,27 +1425,35 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi uuid = get(entry, "uuid", nothing)::Union{String, Nothing} uuid === nothing && continue if UUID(uuid) == pkg.uuid - weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) + weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}} + deps = get(Dict{String, Any}, entry, "deps")::Union{Vector{String}, Dict{String,Any}} + + function expand_deps_list(deps′::Vector{String}) + deps′_expanded = Dict{String, Any}() + for (dep_name, entries) in d + dep_name in deps′ || continue + entries::Vector{Any} + if length(entries) != 1 + error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") + end + entry = first(entries)::Dict{String, Any} + uuid = entry["uuid"]::String + deps′_expanded[dep_name] = uuid + end + return deps′_expanded end - d_weakdeps = Dict{String, Any}() - for (dep_name, entries) in d - dep_name in weakdeps || continue - entries::Vector{Any} - if length(entries) != 1 - error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") - end - entry = first(entries)::Dict{String, Any} - uuid = entry["uuid"]::String - d_weakdeps[dep_name] = uuid + if weakdeps isa Vector{String} + weakdeps = expand_deps_list(weakdeps) + end + if deps isa Vector{String} + deps = expand_deps_list(deps) end - @assert length(d_weakdeps) == length(weakdeps) - return _insert_extension_triggers(pkg, extensions, d_weakdeps) + + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end end end @@ -1454,7 +1461,7 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi return nothing end -function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any}) +function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, totaldeps::Dict{String, Any}) for (ext, triggers) in extensions triggers = triggers::Union{String, Vector{String}} triggers isa String && (triggers = [triggers]) @@ -1468,7 +1475,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} push!(trigger1, gid) for trigger in triggers # TODO: Better error message if this lookup fails? - uuid_trigger = UUID(weakdeps[trigger]::String) + uuid_trigger = UUID(totaldeps[trigger]::String) trigger_id = PkgId(uuid_trigger, trigger) if !haskey(explicit_loaded_modules, trigger_id) || haskey(package_locks, trigger_id) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) diff --git a/base/precompilation.jl b/base/precompilation.jl index 6cf421850a17d..c677b20a4e68a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -401,7 +401,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; all_extdeps_available = true for extdep_uuid in extdep_uuids extdep_name = env.names[extdep_uuid] - if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name)) + if extdep_uuid in keys(env.deps) push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) else all_extdeps_available = false diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 25ebf3475f34c..564ac4679963a 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -351,7 +351,7 @@ Since the primary environment is typically the environment of a project you're w ### [Package Extensions](@id man-extensions) -A package "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of the project file. Those packages can have compat entries like other packages. +A package "extension" is a module that is automatically loaded when a specified set of other packages (its "triggers") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The triggers of an extension are a subset of those packages listed under the `[weakdeps]` (and possibly, but uncommonly the `[deps]`) section of the project file. Those packages can have compat entries like other packages. ```toml name = "MyPackage" @@ -371,8 +371,8 @@ FooExt = "ExtDep" ``` The keys under `extensions` are the names of the extensions. -They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded. -If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity. +They are loaded when all the packages on the right hand side (the triggers) of that extension are loaded. +If an extension only has one trigger the list of triggers can be written as just a string for brevity. The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for extension `FooExt`. The content of an extension is often structured as: @@ -380,10 +380,10 @@ The content of an extension is often structured as: ``` module FooExt -# Load main package and extension dependencies +# Load main package and triggers using MyPackage, ExtDep -# Extend functionality in main package with types from the extension dependencies +# Extend functionality in main package with types from the triggers MyPackage.func(x::ExtDep.SomeStruct) = ... end @@ -391,7 +391,7 @@ end When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections are stored in the manifest file in the section for that package. The dependency lookup rules for -a package are the same as for its "parent" except that the listed extension dependencies are also considered as +a package are the same as for its "parent" except that the listed triggers are also considered as dependencies. ### [Package/Environment Preferences](@id preferences) diff --git a/test/loading.jl b/test/loading.jl index 71d0b07b64376..8d0b76caee998 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1070,6 +1070,9 @@ end using ExtDep2 $ew using ExtDep2 $ew HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set") + using ExtDep3 + $ew using ExtDep3 + $ew HasExtensions.ext_dep_loaded || error("ext_dep_loaded not set") end """ return `$(Base.julia_cmd()) $compile --startup-file=no -e $cmd` @@ -1118,6 +1121,8 @@ end test_ext(HasExtensions, :Extension) using ExtDep2 test_ext(HasExtensions, :ExtensionFolder) + using ExtDep3 + test_ext(HasExtensions, :ExtensionDep) end """ for compile in (`--compiled-modules=no`, ``) diff --git a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml index 8ac961fa1a9a9..004ef7892c173 100644 --- a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml +++ b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0-beta4" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef" +project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f" [[deps.ExtDep]] deps = ["SomePackage"] @@ -18,10 +18,12 @@ version = "0.1.0" [deps.HasExtensions.extensions] Extension = "ExtDep" ExtensionFolder = ["ExtDep", "ExtDep2"] + LinearAlgebraExt = "LinearAlgebra" [deps.HasExtensions.weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.SomePackage]] path = "../SomePackage" diff --git a/test/project/Extensions/ExtDep3.jl/Project.toml b/test/project/Extensions/ExtDep3.jl/Project.toml new file mode 100644 index 0000000000000..690b2f1cffff4 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/Project.toml @@ -0,0 +1,4 @@ +name = "ExtDep3" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" +authors = ["Kristoffer "] diff --git a/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl new file mode 100644 index 0000000000000..96a0b472d06c5 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl @@ -0,0 +1,5 @@ +module ExtDep3 + +greet() = print("Hello World!") + +end # module ExtDep3 diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml index 15f068f250ce3..5706aba59d1e0 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "d523b3401f72a1ed34b7b43749fd2655c6b78542" +project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0" [[deps.ExtDep]] deps = ["SomePackage"] @@ -15,13 +15,20 @@ path = "../ExtDep2" uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" version = "0.1.0" +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" + [[deps.HasExtensions]] +deps = ["ExtDep3"] path = "../HasExtensions.jl" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" [deps.HasExtensions.extensions] Extension = "ExtDep" + ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] LinearAlgebraExt = "LinearAlgebra" diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml index 8f308a9fbee72..aa4956caada74 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml @@ -5,4 +5,5 @@ version = "0.1.0" [deps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" diff --git a/test/project/Extensions/HasExtensions.jl/Manifest.toml b/test/project/Extensions/HasExtensions.jl/Manifest.toml index 55f7958701a75..429c6598fc4f4 100644 --- a/test/project/Extensions/HasExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasExtensions.jl/Manifest.toml @@ -1,7 +1,10 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0-DEV" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "c87947f1f1f070eea848950c304d668a112dec3d" +project_hash = "c0bb526b75939a74a6195ee4819e598918a22ad7" -[deps] +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" diff --git a/test/project/Extensions/HasExtensions.jl/Project.toml b/test/project/Extensions/HasExtensions.jl/Project.toml index a5f9bb1e42d29..fe21a1423f543 100644 --- a/test/project/Extensions/HasExtensions.jl/Project.toml +++ b/test/project/Extensions/HasExtensions.jl/Project.toml @@ -2,6 +2,9 @@ name = "HasExtensions" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" +[deps] +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" + [weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" @@ -9,5 +12,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [extensions] Extension = "ExtDep" +ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] LinearAlgebraExt = "LinearAlgebra" diff --git a/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl new file mode 100644 index 0000000000000..e2710d4d89bbb --- /dev/null +++ b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl @@ -0,0 +1,9 @@ +module ExtensionDep + +using HasExtensions, ExtDep3 + +function __init__() + HasExtensions.ext_dep_loaded = true +end + +end diff --git a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl index dbfaeec4f8812..9d9785f87f790 100644 --- a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl +++ b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl @@ -6,5 +6,6 @@ foo(::HasExtensionsStruct) = 1 ext_loaded = false ext_folder_loaded = false +ext_dep_loaded = false end # module From cb5596bda03260428b3dea599ec0acb03f3149f9 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Sep 2024 12:52:59 +0200 Subject: [PATCH 16/26] prevent loading other extensions when precompiling an extension (#55589) The current way of loading extensions when precompiling an extension very easily leads to cycles. For example, if you have more than one extension and you happen to transitively depend on the triggers of one of your extensions you will immediately hit a cycle where the extensions will try to load each other indefinitely. This is an issue because you cannot directly influence your transitive dependency graph so from this p.o.v the current system of loading extension is "unsound". The test added here checks this scenario and we can now precompile and load it without any warnings or issues. Would have made https://github.com/JuliaLang/julia/issues/55517 a non issue. Fixes https://github.com/JuliaLang/julia/issues/55557 --------- Co-authored-by: KristofferC (cherry picked from commit 4da067167fc414ea4329be3b4fdc516914e102cd) --- base/loading.jl | 16 ++++--- base/precompilation.jl | 47 +------------------ test/loading.jl | 13 +++++ .../Extensions/CyclicExtensions/Manifest.toml | 21 +++++++++ .../Extensions/CyclicExtensions/Project.toml | 13 +++++ .../Extensions/CyclicExtensions/ext/ExtA.jl | 6 +++ .../Extensions/CyclicExtensions/ext/ExtB.jl | 6 +++ .../CyclicExtensions/src/CyclicExtensions.jl | 7 +++ 8 files changed, 76 insertions(+), 53 deletions(-) create mode 100644 test/project/Extensions/CyclicExtensions/Manifest.toml create mode 100644 test/project/Extensions/CyclicExtensions/Project.toml create mode 100644 test/project/Extensions/CyclicExtensions/ext/ExtA.jl create mode 100644 test/project/Extensions/CyclicExtensions/ext/ExtB.jl create mode 100644 test/project/Extensions/CyclicExtensions/src/CyclicExtensions.jl diff --git a/base/loading.jl b/base/loading.jl index 140131b26e7b2..ed35fa929b181 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1350,7 +1350,9 @@ function run_module_init(mod::Module, i::Int=1) end function run_package_callbacks(modkey::PkgId) - run_extension_callbacks(modkey) + if !precompiling_extension + run_extension_callbacks(modkey) + end assert_havelock(require_lock) unlock(require_lock) try @@ -2822,7 +2824,7 @@ end const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, - concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, internal_stderr::IO = stderr, internal_stdout::IO = stdout) + concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, internal_stderr::IO = stderr, internal_stdout::IO = stdout, isext::Bool=false) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists output_o === nothing || rm(output_o, force=true) @@ -2891,7 +2893,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: write(io.in, """ empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated Base.track_nested_precomp($precomp_stack) - Base.precompiling_extension = $(loading_extension) + Base.precompiling_extension = $(loading_extension | isext) Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), $(repr(load_path)), $deps, $(repr(source_path(nothing)))) """) @@ -2948,18 +2950,18 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) +function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), isext::Bool=false) @nospecialize internal_stderr internal_stdout path = locate_package(pkg) path === nothing && throw(ArgumentError("$(repr("text/plain", pkg)) not found during precompilation")) - return compilecache(pkg, path, internal_stderr, internal_stdout; flags, reasons) + return compilecache(pkg, path, internal_stderr, internal_stdout; flags, reasons, isext) end const MAX_NUM_PRECOMPILE_FILES = Ref(10) function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), - reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) + reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), isext::Bool=false) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -2999,7 +3001,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, internal_stderr, internal_stdout) + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, internal_stderr, internal_stdout, isext) if success(p) if cache_objects diff --git a/base/precompilation.jl b/base/precompilation.jl index c677b20a4e68a..406b6dee2f7c8 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -429,51 +429,6 @@ function precompilepkgs(pkgs::Vector{String}=String[]; # consider exts of direct deps to be direct deps so that errors are reported append!(direct_deps, keys(filter(d->last(d) in keys(env.project_deps), exts))) - # An extension effectively depends on another extension if it has all the the - # dependencies of that other extension - function expand_dependencies(depsmap) - function visit!(visited, node, all_deps) - if node in visited - return - end - push!(visited, node) - for dep in get(Set{Base.PkgId}, depsmap, node) - if !(dep in all_deps) - push!(all_deps, dep) - visit!(visited, dep, all_deps) - end - end - end - - depsmap_transitive = Dict{Base.PkgId, Set{Base.PkgId}}() - for package in keys(depsmap) - # Initialize a set to keep track of all dependencies for 'package' - all_deps = Set{Base.PkgId}() - visited = Set{Base.PkgId}() - visit!(visited, package, all_deps) - # Update depsmap with the complete set of dependencies for 'package' - depsmap_transitive[package] = all_deps - end - return depsmap_transitive - end - - depsmap_transitive = expand_dependencies(depsmap) - - for (_, extensions_1) in pkg_exts_map - for extension_1 in extensions_1 - deps_ext_1 = depsmap_transitive[extension_1] - for (_, extensions_2) in pkg_exts_map - for extension_2 in extensions_2 - extension_1 == extension_2 && continue - deps_ext_2 = depsmap_transitive[extension_2] - if issubset(deps_ext_2, deps_ext_1) - push!(depsmap[extension_1], extension_2) - end - end - end - end - end - @debug "precompile: deps collected" # this loop must be run after the full depsmap has been populated for (pkg, pkg_exts) in pkg_exts_map @@ -839,7 +794,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor) do Base.with_logger(Base.NullLogger()) do # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags, cacheflags) + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags, cacheflags, isext = haskey(exts, pkg)) end end if ret isa Base.PrecompilableError diff --git a/test/loading.jl b/test/loading.jl index 8d0b76caee998..08e1eedff7ae3 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1161,6 +1161,19 @@ end finally copy!(LOAD_PATH, old_load_path) end + + # Extension with cycles in dependencies + code = """ + using CyclicExtensions + Base.get_extension(CyclicExtensions, :ExtA) isa Module || error("expected extension to load") + Base.get_extension(CyclicExtensions, :ExtB) isa Module || error("expected extension to load") + CyclicExtensions.greet() + """ + proj = joinpath(@__DIR__, "project", "Extensions", "CyclicExtensions") + cmd = `$(Base.julia_cmd()) --startup-file=no -e $code` + cmd = addenv(cmd, "JULIA_LOAD_PATH" => proj) + @test occursin("Hello Cycles!", String(read(cmd))) + finally try rm(depot_path, force=true, recursive=true) diff --git a/test/project/Extensions/CyclicExtensions/Manifest.toml b/test/project/Extensions/CyclicExtensions/Manifest.toml new file mode 100644 index 0000000000000..a506825cf7995 --- /dev/null +++ b/test/project/Extensions/CyclicExtensions/Manifest.toml @@ -0,0 +1,21 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.4" +manifest_format = "2.0" +project_hash = "ec25ff8df3a5e2212a173c3de2c7d716cc47cd36" + +[[deps.ExtDep]] +deps = ["SomePackage"] +path = "../ExtDep.jl" +uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" +version = "0.1.0" + +[[deps.ExtDep2]] +path = "../ExtDep2" +uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" +version = "0.1.0" + +[[deps.SomePackage]] +path = "../SomePackage" +uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" +version = "0.1.0" diff --git a/test/project/Extensions/CyclicExtensions/Project.toml b/test/project/Extensions/CyclicExtensions/Project.toml new file mode 100644 index 0000000000000..08d539dcc40ae --- /dev/null +++ b/test/project/Extensions/CyclicExtensions/Project.toml @@ -0,0 +1,13 @@ +name = "CyclicExtensions" +uuid = "17d4f0df-b55c-4714-ac4b-55fa23f7355c" +version = "0.1.0" + +[deps] +ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" + +[weakdeps] +SomePackage = "678608ae-7bb3-42c7-98b1-82102067a3d8" + +[extensions] +ExtA = ["SomePackage"] +ExtB = ["SomePackage"] diff --git a/test/project/Extensions/CyclicExtensions/ext/ExtA.jl b/test/project/Extensions/CyclicExtensions/ext/ExtA.jl new file mode 100644 index 0000000000000..fa0c0961633cb --- /dev/null +++ b/test/project/Extensions/CyclicExtensions/ext/ExtA.jl @@ -0,0 +1,6 @@ +module ExtA + +using CyclicExtensions +using SomePackage + +end diff --git a/test/project/Extensions/CyclicExtensions/ext/ExtB.jl b/test/project/Extensions/CyclicExtensions/ext/ExtB.jl new file mode 100644 index 0000000000000..8f6da556d39b8 --- /dev/null +++ b/test/project/Extensions/CyclicExtensions/ext/ExtB.jl @@ -0,0 +1,6 @@ +module ExtB + +using CyclicExtensions +using SomePackage + +end diff --git a/test/project/Extensions/CyclicExtensions/src/CyclicExtensions.jl b/test/project/Extensions/CyclicExtensions/src/CyclicExtensions.jl new file mode 100644 index 0000000000000..f1c2ec2077562 --- /dev/null +++ b/test/project/Extensions/CyclicExtensions/src/CyclicExtensions.jl @@ -0,0 +1,7 @@ +module CyclicExtensions + +using ExtDep + +greet() = print("Hello Cycles!") + +end # module CyclicExtensions From 8f282d356cc661bcf9a15e86abcf00fc278f0825 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 23 Sep 2024 11:39:00 -0400 Subject: [PATCH 17/26] Replace regex package module checks with actual code checks (#55824) Fixes https://github.com/JuliaLang/julia/issues/55792 Replaces https://github.com/JuliaLang/julia/pull/55822 Improves what https://github.com/JuliaLang/julia/pull/51635 was trying to do i.e. ``` ERROR: LoadError: `using/import Printf` outside of a Module detected. Importing a package outside of a module is not allowed during package precompilation. ``` (cherry picked from commit 0fade450a183470b01c656a9001512ef2f1aae47) --- base/loading.jl | 49 ++++++++------------- test/loading.jl | 106 --------------------------------------------- test/precompile.jl | 74 +++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 137 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index ed35fa929b181..afbeeccec3bdd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1489,6 +1489,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} end end +precompiling_package::Bool = false loading_extension::Bool = false precompiling_extension::Bool = false function run_extension_callbacks(extid::ExtensionId) @@ -2181,6 +2182,11 @@ For more details regarding code loading, see the manual sections on [modules](@r [parallel computing](@ref code-availability). """ function require(into::Module, mod::Symbol) + if into === Base.__toplevel__ && precompiling_package + # this error type needs to match the error type compilecache throws for non-125 errors. + error("`using/import $mod` outside of a Module detected. Importing a package outside of a module \ + is not allowed during package precompilation.") + end if _require_world_age[] != typemax(UInt) Base.invoke_in_world(_require_world_age[], __require, into, mod) else @@ -2759,41 +2765,10 @@ function load_path_setup_code(load_path::Bool=true) return code end -""" - check_src_module_wrap(srcpath::String) - -Checks that a package entry file `srcpath` has a module declaration, and that it is before any using/import statements. -""" -function check_src_module_wrap(pkg::PkgId, srcpath::String) - module_rgx = r"^(|end |\"\"\" )\s*(?:@)*(?:bare)?module\s" - load_rgx = r"\b(?:using|import)\s" - load_seen = false - inside_string = false - for s in eachline(srcpath) - if count("\"\"\"", s) == 1 - # ignore module docstrings - inside_string = !inside_string - end - inside_string && continue - if contains(s, module_rgx) - if load_seen - throw(ErrorException("Package $(repr("text/plain", pkg)) source file $srcpath has a using/import before a module declaration.")) - end - return true - end - if startswith(s, load_rgx) - load_seen = true - end - end - throw(ErrorException("Package $(repr("text/plain", pkg)) source file $srcpath does not contain a module declaration.")) -end - # this is called in the external process that generates precompiled package files function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) - check_src_module_wrap(pkg, input) - append!(empty!(Base.DEPOT_PATH), depot_path) append!(empty!(Base.DL_LOAD_PATH), dl_load_path) append!(empty!(Base.LOAD_PATH), load_path) @@ -2820,6 +2795,17 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto finally Core.Compiler.track_newly_inferred.x = false end + # check that the package defined the expected module so we can give a nice error message if not + Base.check_package_module_loaded(pkg) +end + +function check_package_module_loaded(pkg::PkgId) + if !haskey(Base.loaded_modules, pkg) + # match compilecache error type for non-125 errors + error("$(repr("text/plain", pkg)) did not define the expected module `$(pkg.name)`, \ + check for typos in package module name") + end + return nothing end const PRECOMPILE_TRACE_COMPILE = Ref{String}() @@ -2894,6 +2880,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated Base.track_nested_precomp($precomp_stack) Base.precompiling_extension = $(loading_extension | isext) + Base.precompiling_package = true Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), $(repr(load_path)), $deps, $(repr(source_path(nothing)))) """) diff --git a/test/loading.jl b/test/loading.jl index 08e1eedff7ae3..c0f1d04df5ff9 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -855,22 +855,6 @@ end end end -@testset "error message loading pkg bad module name" begin - mktempdir() do tmp - old_loadpath = copy(LOAD_PATH) - try - push!(LOAD_PATH, tmp) - write(joinpath(tmp, "BadCase.jl"), "module badcase end") - @test_logs (:warn, r"The call to compilecache failed.*") match_mode=:any begin - @test_throws ErrorException("package `BadCase` did not define the expected module `BadCase`, \ - check for typos in package module name") (@eval using BadCase) - end - finally - copy!(LOAD_PATH, old_loadpath) - end - end -end - @testset "Preferences loading" begin mktempdir() do dir this_uuid = uuid4() @@ -1282,96 +1266,6 @@ end @test success(`$(Base.julia_cmd()) --startup-file=no -e 'using Statistics'`) end -@testset "checking srcpath modules" begin - p = Base.PkgId("Dummy") - fpath, _ = mktemp() - @testset "valid" begin - write(fpath, """ - module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - baremodule Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - \"\"\" - Foo - using Foo - \"\"\" - module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - \"\"\" Foo \"\"\" - module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - \"\"\" - Foo - \"\"\" module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - @doc let x = 1 - x - end module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - # using foo - module Foo - using Bar - end - """) - @test Base.check_src_module_wrap(p, fpath) - end - @testset "invalid" begin - write(fpath, """ - # module Foo - using Bar - # end - """) - @test_throws ErrorException Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - using Bar - module Foo - end - """) - @test_throws ErrorException Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - using Bar - """) - @test_throws ErrorException Base.check_src_module_wrap(p, fpath) - - write(fpath, """ - x = 1 - """) - @test_throws ErrorException Base.check_src_module_wrap(p, fpath) - end -end - @testset "relocatable upgrades #51989" begin mktempdir() do depot project_path = joinpath(depot, "project") diff --git a/test/precompile.jl b/test/precompile.jl index 65b94e81c31ab..4dfa5f672072f 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2164,6 +2164,80 @@ precompile_test_harness("Issue #52063") do load_path end end +precompile_test_harness("Detecting importing outside of a package module") do load_path + io = IOBuffer() + write(joinpath(load_path, "ImportBeforeMod.jl"), + """ + import Printf + module ImportBeforeMod + end #module + """) + @test_throws r"Failed to precompile ImportBeforeMod" Base.compilecache(Base.identify_package("ImportBeforeMod"), io, io) + @test occursin( + "`using/import Printf` outside of a Module detected. Importing a package outside of a module is not allowed during package precompilation.", + String(take!(io))) + + + write(joinpath(load_path, "HarmlessComments.jl"), + """ + # import Printf + #= + import Printf + =# + module HarmlessComments + end #module + # import Printf + #= + import Printf + =# + """) + Base.compilecache(Base.identify_package("HarmlessComments")) + + + write(joinpath(load_path, "ImportAfterMod.jl"), """ + module ImportAfterMod + end #module + import Printf + """) + @test_throws r"Failed to precompile ImportAfterMod" Base.compilecache(Base.identify_package("ImportAfterMod"), io, io) + @test occursin( + "`using/import Printf` outside of a Module detected. Importing a package outside of a module is not allowed during package precompilation.", + String(take!(io))) +end + +precompile_test_harness("No package module") do load_path + io = IOBuffer() + write(joinpath(load_path, "NoModule.jl"), + """ + 1 + """) + @test_throws r"Failed to precompile NoModule" Base.compilecache(Base.identify_package("NoModule"), io, io) + @test occursin( + "NoModule [top-level] did not define the expected module `NoModule`, check for typos in package module name", + String(take!(io))) + + + write(joinpath(load_path, "WrongModuleName.jl"), + """ + module DifferentName + x = 1 + end #module + """) + @test_throws r"Failed to precompile WrongModuleName" Base.compilecache(Base.identify_package("WrongModuleName"), io, io) + @test occursin( + "WrongModuleName [top-level] did not define the expected module `WrongModuleName`, check for typos in package module name", + String(take!(io))) + + + write(joinpath(load_path, "NoModuleWithImport.jl"), """ + import Printf + """) + @test_throws r"Failed to precompile NoModuleWithImport" Base.compilecache(Base.identify_package("NoModuleWithImport"), io, io) + @test occursin( + "`using/import Printf` outside of a Module detected. Importing a package outside of a module is not allowed during package precompilation.", + String(take!(io))) +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH) From 0628fa0d42963ce10646410e15db5717da08c6d1 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 14 Sep 2024 12:22:31 -0400 Subject: [PATCH 18/26] Add a docs section about loading/precomp/ttfx time tuning (#55569) (cherry picked from commit 243bdede3d686b1eaa634db190f2bab3e5f4de72) --- doc/src/manual/performance-tips.md | 105 +++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index bdceb5a0c2493..040fb9f8bba5f 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -1722,3 +1722,108 @@ Prominent examples include [MKL.jl](https://github.com/JuliaLinearAlgebra/MKL.jl These are external packages, so we will not discuss them in detail here. Please refer to their respective documentations (especially because they have different behaviors than OpenBLAS with respect to multithreading). + +## Execution latency, package loading and package precompiling time + +### Reducing time to first plot etc. + +The first time a julia method is called it (and any methods it calls, or ones that can be statically determined) will be +compiled. The [`@time`](@ref) macro family illustrates this. + +``` +julia> foo() = rand(2,2) * rand(2,2) +foo (generic function with 1 method) + +julia> @time @eval foo(); + 0.252395 seconds (1.12 M allocations: 56.178 MiB, 2.93% gc time, 98.12% compilation time) + +julia> @time @eval foo(); + 0.000156 seconds (63 allocations: 2.453 KiB) +``` + +Note that `@time @eval` is better for measuring compilation time because without [`@eval`](@ref), some compilation may +already be done before timing starts. + +When developing a package, you may be able to improve the experience of your users with *precompilation* +so that when they use the package, the code they use is already compiled. To precompile package code effectively, it's +recommended to use [`PrecompileTools.jl`](https://julialang.github.io/PrecompileTools.jl/stable/) to run a +"precompile workload" during precompilation time that is representative of typical package usage, which will cache the +native compiled code into the package `pkgimage` cache, greatly reducing "time to first execution" (often referred to as +TTFX) for such usage. + +Note that [`PrecompileTools.jl`](https://julialang.github.io/PrecompileTools.jl/stable/) workloads can be +disabled and sometimes configured via Preferences if you do not want to spend the extra time precompiling, which +may be the case during development of a package. + +### Reducing package loading time + +Keeping the time taken to load the package down is usually helpful. +General good practice for package developers includes: + +1. Reduce your dependencies to those you really need. Consider using [package extensions](@ref) to support interoperability with other packages without bloating your essential dependencies. +3. Avoid use of [`__init__()`](@ref) functions unless there is no alternative, especially those which might trigger a lot + of compilation, or just take a long time to execute. +4. Where possible, fix [invalidations](https://julialang.org/blog/2020/08/invalidations/) among your dependencies and from your package code. + +The tool [`@time_imports`](@ref) can be useful in the REPL to review the above factors. + +```julia-repl +julia> @time @time_imports using Plots + 0.5 ms Printf + 16.4 ms Dates + 0.7 ms Statistics + ┌ 23.8 ms SuiteSparse_jll.__init__() 86.11% compilation time (100% recompilation) + 90.1 ms SuiteSparse_jll 91.57% compilation time (82% recompilation) + 0.9 ms Serialization + ┌ 39.8 ms SparseArrays.CHOLMOD.__init__() 99.47% compilation time (100% recompilation) + 166.9 ms SparseArrays 23.74% compilation time (100% recompilation) + 0.4 ms Statistics → SparseArraysExt + 0.5 ms TOML + 8.0 ms Preferences + 0.3 ms PrecompileTools + 0.2 ms Reexport +... many deps omitted for example ... + 1.4 ms Tar + ┌ 73.8 ms p7zip_jll.__init__() 99.93% compilation time (100% recompilation) + 79.4 ms p7zip_jll 92.91% compilation time (100% recompilation) + ┌ 27.7 ms GR.GRPreferences.__init__() 99.77% compilation time (100% recompilation) + 43.0 ms GR 64.26% compilation time (100% recompilation) + ┌ 2.1 ms Plots.__init__() 91.80% compilation time (100% recompilation) + 300.9 ms Plots 0.65% compilation time (100% recompilation) + 1.795602 seconds (3.33 M allocations: 190.153 MiB, 7.91% gc time, 39.45% compilation time: 97% of which was recompilation) + +``` + +Notice that in this example there are multiple packages loaded, some with `__init__()` functions, some of which cause +compilation of which some is recompilation. Recompilation is caused by earlier packages invalidating methods, then in +these cases when the following packages run their `__init__()` function some hit recompilation before the code can be run. + +Further, note the `Statistics` extension `SparseArraysExt` has been activated because `SparseArrays` is in the dependency +tree. i.e. see `0.4 ms Statistics → SparseArraysExt`. + +This report gives a good opportunity to review whether the cost of dependency load time is worth the functionality it brings. +Also the `Pkg` utility `why` can be used to report why a an indirect dependency exists. + +``` +(CustomPackage) pkg> why FFMPEG_jll + Plots → FFMPEG → FFMPEG_jll + Plots → GR → GR_jll → FFMPEG_jll +``` + +or to see the indirect dependencies that a package brings in, you can `pkg> rm` the package, see the deps that are removed +from the manifest, then revert the change with `pkg> undo`. + +If loading time is dominated by slow `__init__()` methods having compilation, one verbose way to identify what is being +compiled is to use the julia args `--trace-compile=stderr` which will report a [`precompile`](@ref) +statement each time a method is compiled. For instance, the full setup would be: + +``` +$ julia --startup-file=no --trace-compile=stderr +julia> @time @time_imports using CustomPackage +... +``` + +Note the `--startup-file=no` which helps isolate the test from packages you may have in your `startup.jl`. + +More analysis of the reasons for recompilation can be achieved with the +[`SnoopCompile`](https://github.com/timholy/SnoopCompile.jl) package. From 0de7b6c6a24900560d8d2b98dc33dd5e920d9d26 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Oct 2024 12:13:55 +0200 Subject: [PATCH 19/26] =?UTF-8?q?Revert=20"do=20not=20intentionally=20supp?= =?UTF-8?q?ress=20errors=20in=20precompile=20script=20from=20being=20?= =?UTF-8?q?=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 4a4ca9c815207a80ea81b884b196dfeafc3cb877. (cherry picked from commit d65f48a58a0a35ba48e84ee83535903fa793309d) --- contrib/generate_precompile.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 3aeb2d5b2c0ad..e4eeeed577686 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -342,7 +342,8 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step1" => "F$n_step1") return :ok end - PARALLEL_PRECOMPILATION ? bind(statements_step1, step1) : wait(step1) + Base.errormonitor(step1) + !PARALLEL_PRECOMPILATION && wait(step1) # Create a staging area where all the loaded packages are available PrecompileStagingArea = Module() @@ -356,7 +357,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe # Make statements unique statements = Set{String}() # Execute the precompile statements - for statement in statements_step1 + for sts in [statements_step1,], statement in sts # Main should be completely clean occursin("Main.", statement) && continue Base.in!(statement, statements) && continue @@ -392,7 +393,6 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe println() # Seems like a reasonable number right now, adjust as needed # comment out if debugging script - have_repl = false n_succeeded > (have_repl ? 650 : 90) || @warn "Only $n_succeeded precompile statements" fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") @@ -403,6 +403,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe finally fancyprint && print(ansi_enablecursor) GC.gc(true); GC.gc(false); # reduce memory footprint + return end generate_precompile_statements() From c092d367ca978fbbeac118680025270d1c92a5fe Mon Sep 17 00:00:00 2001 From: Daniel Pinyol Date: Thu, 10 Oct 2024 21:53:47 +0200 Subject: [PATCH 20/26] backport 1.11: fix `_growbeg!` unncessary resizing (#56029) (#56090) This was very explicitly designed such that if there was a bunch of extra space at the end of the array, we would copy rather than allocating, but by making `newmemlen` be at least `overallocation(memlen)` rather than `overallocation(len)`, this branch was never hit. found by https://github.com/JuliaLang/julia/issues/56026 (cherry picked from commit d4ca92c2718c952d1bc2807f1096da3b02a4852a) Co-authored-by: Oscar Smith --- base/array.jl | 9 +++++++-- test/arrayops.jl | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index d7abb2a757454..762cf2cbc5bc5 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1062,10 +1062,11 @@ function _growbeg!(a::Vector, delta::Integer) setfield!(a, :ref, @inbounds memoryref(ref, 1 - delta)) else @noinline (function() + @_terminates_locally_meta memlen = length(mem) # since we will allocate the array in the middle of the memory we need at least 2*delta extra space # the +1 is because I didn't want to have an off by 1 error. - newmemlen = max(overallocation(memlen), len + 2 * delta + 1) + newmemlen = max(overallocation(len), len + 2 * delta + 1) newoffset = div(newmemlen - newlen, 2) + 1 # If there is extra data after the end of the array we can use that space so long as there is enough # space at the end that there won't be quadratic behavior with a mix of growth from both ends. @@ -1074,10 +1075,14 @@ function _growbeg!(a::Vector, delta::Integer) if newoffset + newlen < memlen newoffset = div(memlen - newlen, 2) + 1 newmem = mem + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) + for j in offset:newoffset+delta-1 + @inbounds _unsetindex!(mem, j) + end else newmem = array_new_memory(mem, newmemlen) + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) end - unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) setfield!(a, :ref, @inbounds memoryref(newmem, newoffset)) end)() end diff --git a/test/arrayops.jl b/test/arrayops.jl index d86faf6f96004..5dfc9a74887ec 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -467,6 +467,11 @@ end @test vc == [v[1:(i-1)]; 5; v[i:end]] end @test_throws BoundsError insert!(v, 5, 5) + + # test that data is copied when there is plenty of room to do so + v = empty!(collect(1:100)) + pushfirst!(v, 1) + @test length(v.ref.mem) == 100 end @testset "popat!(::Vector, i, [default])" begin From 1fa66d854a4983760fe979e06139a0fc4286cca2 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Sun, 3 Mar 2024 04:14:06 +0100 Subject: [PATCH 21/26] Provide better error hint when `UndefVarError` results from name clashes (#53469) We can detect this since we set the `usingfailed` bit when the clash occurs (to avoid printing the `WARNING` multiple times). In this case, typos or missing imports (the current message) isn't quite as clear as it could be, because in fact the name is probably spelled totally right, it's just that there is a missing explicit import or the name should be qualified. This code will stop working if we change the flags in `Core.Binding`, but the test I added should catch that. However if REPL is supposed to be independent of Base and not depend on internals there, there could be an issue. In that case we should probably add an API to Base to inspect this `usingfailed` bit so we can use it in the REPL. --------- Co-authored-by: Jameson Nash Co-authored-by: Alex Arslan (cherry picked from commit 0f902bf2f7ea6dcf0955b5dc37f00a2c1447af28) --- stdlib/REPL/src/REPL.jl | 11 ++++++++++- stdlib/REPL/test/repl.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 0d87096b81e21..2f80b02d2f876 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -44,7 +44,16 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) if C_NULL == owner # No global of this name exists in this module. # This is the common case, so do not print that information. - print(io, "\nSuggestion: check for spelling errors or missing imports.") + # It could be the binding was exported by two modules, which we can detect + # by the `usingfailed` flag in the binding: + if isdefined(bnd, :flags) && Bool(bnd.flags >> 4 & 1) # magic location of the `usingfailed` flag + print(io, "\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + else + print(io, "\nSuggestion: check for spelling errors or missing imports.") + end owner = bnd else owner = unsafe_pointer_to_objref(owner)::Core.Binding diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index b5007c64c72aa..debf7e5ad12e4 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1698,6 +1698,35 @@ finally empty!(Base.Experimental._hint_handlers) end +try # test the functionality of `UndefVarError_hint` against import clashes + @assert isempty(Base.Experimental._hint_handlers) + Base.Experimental.register_error_hint(REPL.UndefVarError_hint, UndefVarError) + + @eval module X + + module A + export x + x = 1 + end # A + + module B + export x + x = 2 + end # B + + using .A, .B + + end # X + + expected_message = string("\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + @test_throws expected_message X.x +finally + empty!(Base.Experimental._hint_handlers) +end + # Hints for tab completes fake_repl() do stdin_write, stdout_read, repl From 4751c230eb518d73578d644a05f219b6bfb678cc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 16 Aug 2024 14:14:36 -0400 Subject: [PATCH 22/26] fix overlapping definitions of `Base.active_module` and `REPL.active_module` (#55316) also avoid calling `active_module` from low-level printing functions fix #54888 (cherry picked from commit ddecfe74ffdfbbdfe0c2e78cee059470982e0ab9) --- base/Enums.jl | 2 +- base/show.jl | 44 +++++++++++++++++------------------------ stdlib/REPL/src/REPL.jl | 18 +++++++---------- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index 6e9efd8ccde38..d4094945853ec 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -44,7 +44,7 @@ Base.print(io::IO, x::Enum) = print(io, _symbol(x)) function Base.show(io::IO, x::Enum) sym = _symbol(x) if !(get(io, :compact, false)::Bool) - from = get(io, :module, Base.active_module()) + from = get(io, :module, Main) def = parentmodule(typeof(x)) if from === nothing || !Base.isvisible(sym, def, from) show(io, def) diff --git a/base/show.jl b/base/show.jl index 13694df898def..b9cda8d64ec36 100644 --- a/base/show.jl +++ b/base/show.jl @@ -514,24 +514,16 @@ function _show_default(io::IO, @nospecialize(x)) end function active_module() - isassigned(REPL_MODULE_REF) || return Main - REPL = REPL_MODULE_REF[] - return invokelatest(REPL.active_module)::Module + if ccall(:jl_is_in_pure_context, Bool, ()) + error("active_module() should not be called from a pure context") + end + if !@isdefined(active_repl) || active_repl === nothing + return Main + end + return invokelatest(active_module, active_repl)::Module end -# Check if a particular symbol is exported from a standard library module -function is_exported_from_stdlib(name::Symbol, mod::Module) - !isdefined(mod, name) && return false - orig = getfield(mod, name) - while !(mod === Base || mod === Core) - activemod = active_module() - parent = parentmodule(mod) - if mod === activemod || mod === parent || parent === activemod - return false - end - mod = parent - end - return isexported(mod, name) && isdefined(mod, name) && !isdeprecated(mod, name) && getfield(mod, name) === orig +module UsesCoreAndBaseOnly end function show_function(io::IO, f::Function, compact::Bool, fallback::Function) @@ -544,13 +536,13 @@ function show_function(io::IO, f::Function, compact::Bool, fallback::Function) print(io, mt.name) elseif isdefined(mt, :module) && isdefined(mt.module, mt.name) && getfield(mt.module, mt.name) === f - mod = active_module() - if is_exported_from_stdlib(mt.name, mt.module) || mt.module === mod - show_sym(io, mt.name) - else + # this used to call the removed internal function `is_exported_from_stdlib`, which effectively + # just checked for exports from Core and Base. + mod = get(io, :module, UsesCoreAndBaseOnly) + if !(isvisible(mt.name, mt.module, mod) || mt.module === mod) print(io, mt.module, ".") - show_sym(io, mt.name) end + show_sym(io, mt.name) else fallback(io, f) end @@ -737,9 +729,9 @@ end function show_typealias(io::IO, name::GlobalRef, x::Type, env::SimpleVector, wheres::Vector) if !(get(io, :compact, false)::Bool) # Print module prefix unless alias is visible from module passed to - # IOContext. If :module is not set, default to Main (or current active module). + # IOContext. If :module is not set, default to Main. # nothing can be used to force printing prefix. - from = get(io, :module, active_module()) + from = get(io, :module, Main) if (from === nothing || !isvisible(name.name, name.mod, from)) show(io, name.mod) print(io, ".") @@ -1053,9 +1045,9 @@ function show_type_name(io::IO, tn::Core.TypeName) quo = false if !(get(io, :compact, false)::Bool) # Print module prefix unless type is visible from module passed to - # IOContext If :module is not set, default to Main (or current active module). + # IOContext If :module is not set, default to Main. # nothing can be used to force printing prefix - from = get(io, :module, active_module()) + from = get(io, :module, Main) if isdefined(tn, :module) && (from === nothing || !isvisible(sym, tn.module, from)) show(io, tn.module) print(io, ".") @@ -2525,7 +2517,7 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg uw = unwrap_unionall(ft) if ft <: Function && isa(uw, DataType) && isempty(uw.parameters) && _isself(uw) uwmod = parentmodule(uw) - if qualified && !is_exported_from_stdlib(uw.name.mt.name, uwmod) && uwmod !== Main + if qualified && !isexported(uwmod, uw.name.mt.name) && uwmod !== Main print_within_stacktrace(io, uwmod, '.', bold=true) end s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 2f80b02d2f876..e49d76e0d56bf 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -351,7 +351,7 @@ end function display(d::REPLDisplay, mime::MIME"text/plain", x) x = Ref{Any}(x) with_repl_linfo(d.repl) do io - io = IOContext(io, :limit => true, :module => active_module(d)::Module) + io = IOContext(io, :limit => true, :module => Base.active_module(d)::Module) if d.repl isa LineEditREPL mistate = d.repl.mistate mode = LineEdit.mode(mistate) @@ -374,7 +374,7 @@ display(d::REPLDisplay, x) = display(d, MIME("text/plain"), x) function print_response(repl::AbstractREPL, response, show_value::Bool, have_color::Bool) repl.waserror = response[2] with_repl_linfo(repl) do io - io = IOContext(io, :module => active_module(repl)::Module) + io = IOContext(io, :module => Base.active_module(repl)::Module) print_response(io, response, show_value, have_color, specialdisplay(repl)) end return nothing @@ -475,7 +475,7 @@ function run_repl(repl::AbstractREPL, @nospecialize(consumer = x -> nothing); ba Core.println(Core.stderr, e) Core.println(Core.stderr, catch_backtrace()) end - get_module = () -> active_module(repl) + get_module = () -> Base.active_module(repl) if backend_on_current_task t = @async run_frontend(repl, backend_ref) errormonitor(t) @@ -607,13 +607,9 @@ REPLCompletionProvider() = REPLCompletionProvider(LineEdit.Modifiers()) mutable struct ShellCompletionProvider <: CompletionProvider end struct LatexCompletions <: CompletionProvider end -function active_module() # this method is also called from Base - isdefined(Base, :active_repl) || return Main - return active_module(Base.active_repl::AbstractREPL) -end -active_module((; mistate)::LineEditREPL) = mistate === nothing ? Main : mistate.active_module -active_module(::AbstractREPL) = Main -active_module(d::REPLDisplay) = active_module(d.repl) +Base.active_module((; mistate)::LineEditREPL) = mistate === nothing ? Main : mistate.active_module +Base.active_module(::AbstractREPL) = Main +Base.active_module(d::REPLDisplay) = Base.active_module(d.repl) setmodifiers!(c::CompletionProvider, m::LineEdit.Modifiers) = nothing @@ -1052,7 +1048,7 @@ enable_promptpaste(v::Bool) = JL_PROMPT_PASTE[] = v function contextual_prompt(repl::LineEditREPL, prompt::Union{String,Function}) function () - mod = active_module(repl) + mod = Base.active_module(repl) prefix = mod == Main ? "" : string('(', mod, ") ") pr = prompt isa String ? prompt : prompt() prefix * pr From 4cf39756e9e0b3a0729f4a0facec7a4c22e52ae6 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 11 Oct 2024 01:57:36 -0400 Subject: [PATCH 23/26] Remove warning from c when binding is ambiguous (#56103) (cherry picked from commit 1438b1578941d0f1cc8f8c958cf3bd2927fd482c) --- doc/src/manual/modules.md | 4 ++-- src/module.c | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index 8c366616bac49..75e79d889dc7e 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -281,14 +281,14 @@ julia> module B B ``` -The statement `using .A, .B` works, but when you try to call `f`, you get a warning +The statement `using .A, .B` works, but when you try to call `f`, you get an error with a hint ```jldoctest module_manual julia> using .A, .B julia> f -WARNING: both B and A export "f"; uses of it in module Main must be qualified ERROR: UndefVarError: `f` not defined in `Main` +Hint: It looks like two or more modules export different bindings with this name, resulting in ambiguity. Try explicitly importing it from a particular module, or qualifying the name with the module it should come from. ``` Here, Julia cannot decide which `f` you are referring to, so you have to make a choice. The following solutions are commonly used: diff --git a/src/module.c b/src/module.c index f33c9bc4f31d3..cb19c87d10ef6 100644 --- a/src/module.c +++ b/src/module.c @@ -336,11 +336,6 @@ static jl_binding_t *using_resolve_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl // the owner will still be NULL, so it can be later imported or defined tempb = jl_get_module_binding(m, var, 1); tempb->usingfailed = 1; - jl_printf(JL_STDERR, - "WARNING: both %s and %s export \"%s\"; uses of it in module %s must be qualified\n", - jl_symbol_name(owner->name), - jl_symbol_name(imp->name), jl_symbol_name(var), - jl_symbol_name(m->name)); } return NULL; } From 788775de4939436730f12c229df369e393cd1772 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:27:10 +0900 Subject: [PATCH 24/26] improve `allunique`'s type stability (#56161) Caught by https://github.com/aviatesk/JET.jl/issues/667. --- base/set.jl | 4 ++-- test/sets.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/set.jl b/base/set.jl index 2f96cef626b6f..d1f9458039cd4 100644 --- a/base/set.jl +++ b/base/set.jl @@ -549,8 +549,8 @@ function allunique(A::StridedArray) if length(A) < 32 _indexed_allunique(A) elseif OrderStyle(eltype(A)) === Ordered() - a1, rest1 = Iterators.peel(A) - a2, rest = Iterators.peel(rest1) + a1, rest1 = Iterators.peel(A)::Tuple{Any,Any} + a2, rest = Iterators.peel(rest1)::Tuple{Any,Any} if !isequal(a1, a2) compare = isless(a1, a2) ? isless : (a,b) -> isless(b,a) for a in rest diff --git a/test/sets.jl b/test/sets.jl index 4ab360c9fedd4..b78d2f15dd989 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -644,6 +644,7 @@ end @test !allunique((NaN, NaN)) # Known length 1, need not evaluate: @test allunique(error(x) for x in [1]) + # @test_opt allunique(Int[]) end @testset "allunique(f, xs)" begin From 66b620f2619365c9a56f1c71673b7288af78bf5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Sun, 6 Oct 2024 23:10:51 +0100 Subject: [PATCH 25/26] [LinearAlgebra] Remove unreliable doctests (#56011) The exact textual representation of the output of these doctests depend on the specific kernel used by the BLAS backend, and can vary between versions of OpenBLAS (as it did in #41973), or between different CPUs, which makes these doctests unreliable. Fix #55998. (cherry picked from commit c2a2e38079c1b540e93e8524cf53da4153358481) --- stdlib/LinearAlgebra/src/LinearAlgebra.jl | 26 ++++------------------- stdlib/LinearAlgebra/src/hessenberg.jl | 2 +- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index ec0363e1f751c..867236b2bdffb 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -360,17 +360,8 @@ julia> Y = zero(X); julia> ldiv!(Y, qr(A), X); -julia> Y -3-element Vector{Float64}: - 0.7128099173553719 - -0.051652892561983674 - 0.10020661157024757 - -julia> A\\X -3-element Vector{Float64}: - 0.7128099173553719 - -0.05165289256198333 - 0.10020661157024785 +julia> Y ≈ A\\X +true ``` """ ldiv!(Y, A, B) @@ -401,17 +392,8 @@ julia> Y = copy(X); julia> ldiv!(qr(A), X); -julia> X -3-element Vector{Float64}: - 0.7128099173553719 - -0.051652892561983674 - 0.10020661157024757 - -julia> A\\Y -3-element Vector{Float64}: - 0.7128099173553719 - -0.05165289256198333 - 0.10020661157024785 +julia> X ≈ A\\Y +true ``` """ ldiv!(A, B) diff --git a/stdlib/LinearAlgebra/src/hessenberg.jl b/stdlib/LinearAlgebra/src/hessenberg.jl index f20d76e727b34..c3f97766b2493 100644 --- a/stdlib/LinearAlgebra/src/hessenberg.jl +++ b/stdlib/LinearAlgebra/src/hessenberg.jl @@ -441,7 +441,7 @@ This is useful because multiple shifted solves `(F + μ*I) \\ b` Iterating the decomposition produces the factors `F.Q, F.H, F.μ`. # Examples -```jldoctest +```julia-repl julia> A = [4. 9. 7.; 4. 4. 1.; 4. 3. 2.] 3×3 Matrix{Float64}: 4.0 9.0 7.0 From 4e03986f3f070aa1645ad79f675b96c9c3bfba1c Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Tue, 15 Oct 2024 14:50:08 +0530 Subject: [PATCH 26/26] Fix zero elements for block-matrix kron involving Diagonal (#55941) --- stdlib/LinearAlgebra/src/diagonal.jl | 70 ++++++++++++++++++++++++--- stdlib/LinearAlgebra/test/diagonal.jl | 10 ++++ 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index 830332db0dbb1..eb0aec7726bc2 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -634,16 +634,33 @@ for Tri in (:UpperTriangular, :LowerTriangular) end @inline function kron!(C::AbstractMatrix, A::Diagonal, B::Diagonal) - valA = A.diag; nA = length(valA) - valB = B.diag; nB = length(valB) + valA = A.diag; mA, nA = size(A) + valB = B.diag; mB, nB = size(B) nC = checksquare(C) @boundscheck nC == nA*nB || throw(DimensionMismatch(lazy"expect C to be a $(nA*nB)x$(nA*nB) matrix, got size $(nC)x$(nC)")) - isempty(A) || isempty(B) || fill!(C, zero(A[1,1] * B[1,1])) + zerofilled = false + if !(isempty(A) || isempty(B)) + z = A[1,1] * B[1,1] + if haszero(typeof(z)) + # in this case, the zero is unique + fill!(C, zero(z)) + zerofilled = true + end + end @inbounds for i = 1:nA, j = 1:nB idx = (i-1)*nB+j C[idx, idx] = valA[i] * valB[j] end + if !zerofilled + for j in 1:nA, i in 1:mA + Δrow, Δcol = (i-1)*mB, (j-1)*nB + for k in 1:nB, l in 1:mB + i == j && k == l && continue + C[Δrow + l, Δcol + k] = A[i,j] * B[l,k] + end + end + end return C end @@ -670,7 +687,15 @@ end (mC, nC) = size(C) @boundscheck (mC, nC) == (mA * mB, nA * nB) || throw(DimensionMismatch(lazy"expect C to be a $(mA * mB)x$(nA * nB) matrix, got size $(mC)x$(nC)")) - isempty(A) || isempty(B) || fill!(C, zero(A[1,1] * B[1,1])) + zerofilled = false + if !(isempty(A) || isempty(B)) + z = A[1,1] * B[1,1] + if haszero(typeof(z)) + # in this case, the zero is unique + fill!(C, zero(z)) + zerofilled = true + end + end m = 1 @inbounds for j = 1:nA A_jj = A[j,j] @@ -681,6 +706,18 @@ end end m += (nA - 1) * mB end + if !zerofilled + # populate the zero elements + for i in 1:mA + i == j && continue + A_ij = A[i, j] + Δrow, Δcol = (i-1)*mB, (j-1)*nB + for k in 1:nB, l in 1:nA + B_lk = B[l, k] + C[Δrow + l, Δcol + k] = A_ij * B_lk + end + end + end m += mB end return C @@ -693,17 +730,36 @@ end (mC, nC) = size(C) @boundscheck (mC, nC) == (mA * mB, nA * nB) || throw(DimensionMismatch(lazy"expect C to be a $(mA * mB)x$(nA * nB) matrix, got size $(mC)x$(nC)")) - isempty(A) || isempty(B) || fill!(C, zero(A[1,1] * B[1,1])) + zerofilled = false + if !(isempty(A) || isempty(B)) + z = A[1,1] * B[1,1] + if haszero(typeof(z)) + # in this case, the zero is unique + fill!(C, zero(z)) + zerofilled = true + end + end m = 1 @inbounds for j = 1:nA for l = 1:mB Bll = B[l,l] - for k = 1:mA - C[m] = A[k,j] * Bll + for i = 1:mA + C[m] = A[i,j] * Bll m += nB end m += 1 end + if !zerofilled + for i in 1:mA + A_ij = A[i, j] + Δrow, Δcol = (i-1)*mB, (j-1)*nB + for k in 1:nB, l in 1:mB + l == k && continue + B_lk = B[l, k] + C[Δrow + l, Δcol + k] = A_ij * B_lk + end + end + end m -= nB end return C diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 7049dd784faa8..cfd8a5277e5f0 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -1323,4 +1323,14 @@ end @test checkbounds(Bool, D, diagind(D, IndexCartesian())) end +@testset "zeros in kron with block matrices" begin + D = Diagonal(1:2) + B = reshape([ones(2,2), ones(3,2), ones(2,3), ones(3,3)], 2, 2) + @test kron(D, B) == kron(Array(D), B) + @test kron(B, D) == kron(B, Array(D)) + D2 = Diagonal([ones(2,2), ones(3,3)]) + @test kron(D, D2) == Diagonal([diag(D2); 2diag(D2)]) + @test kron(D2, D) == Diagonal([ones(2,2), fill(2.0,2,2), ones(3,3), fill(2.0,3,3)]) +end + end # module TestDiagonal