Skip to content

Commit 6590764

Browse files
NHDalyNHDaly
authored
Add wall-time profiles to ProfileEndpoints.jl (#50)
* Add wall-time profiles to ProfileEndpoints.jl - Through the HTTP endpoints: - /profile_wall - /profile_wall_start - /profile_wall_stop - Through the debug super-endpoint: - /debug_engine profile_type="wall_profile" - /debug_engine profile_type="wall_profile_start" - /debug_engine profile_type="wall_profile_stop" * Fix broken test, from #48 * Bump to 2.6.0 * remove CI support for 1.9
1 parent 205054c commit 6590764

File tree

5 files changed

+238
-56
lines changed

5 files changed

+238
-56
lines changed

‎.github/workflows/CI.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,8 @@ jobs:
1313
fail-fast: false
1414
matrix:
1515
version:
16-
- "1.6" # LTS
16+
- "1.6" # min-supported version
1717
- "1" # Latest
18-
- "~1.9.0-0" # To test heap snapshot; remove when v1.9 is latest Julia release.
1918
- "nightly"
2019
os:
2120
- ubuntu-latest

‎Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "ProfileEndpoints"
22
uuid = "873a18e9-432f-47dd-82a7-1a805cf6f852"
33
authors = ["Nathan Daly <[email protected]>", "Dana Wilson <[email protected]>"]
4-
version = "2.5.0"
4+
version = "2.6.0"
55

66
[deps]
77
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"

‎src/ProfileEndpoints.jl

Lines changed: 82 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,17 @@ function _http_create_response_with_profile_as_file(filename)
3434
end
3535

3636
###
37-
### CPU
37+
### Task Profiles: CPU profiles + Wall profiles
3838
###
3939

40+
@enum TaskProfileType CPU_PROFILE WALL_PROFILE
41+
4042
default_n() = "1e8"
4143
default_delay() = "0.01"
4244
default_duration() = "10.0"
4345
default_pprof() = "true"
4446

45-
cpu_profile_error_message() = """Need to provide query params:
47+
profile_error_message() = """Need to provide query params:
4648
- duration=$(default_duration())
4749
- delay=$(default_delay())
4850
- n=$(default_n())
@@ -67,80 +69,116 @@ The default `n` is 1e8, which should be big enough for most profiles.
6769
"""
6870

6971
function cpu_profile_endpoint(req::HTTP.Request)
72+
profile_endpoint(CPU_PROFILE, req)
73+
end
74+
75+
function cpu_profile_start_endpoint(req::HTTP.Request)
76+
profile_start_endpoint(CPU_PROFILE, req)
77+
end
78+
79+
function cpu_profile_stop_endpoint(req::HTTP.Request)
80+
profile_stop_endpoint(CPU_PROFILE, req)
81+
end
82+
83+
function wall_profile_endpoint(req::HTTP.Request)
84+
profile_endpoint(WALL_PROFILE, req)
85+
end
86+
87+
function wall_profile_start_endpoint(req::HTTP.Request)
88+
profile_start_endpoint(WALL_PROFILE, req)
89+
end
90+
91+
function wall_profile_stop_endpoint(req::HTTP.Request)
92+
profile_stop_endpoint(WALL_PROFILE, req)
93+
end
94+
95+
function profile_endpoint(type::TaskProfileType, req::HTTP.Request)
7096
uri = HTTP.URI(req.target)
7197
qp = HTTP.queryparams(uri)
7298
if isempty(qp)
7399
@info "TODO: interactive HTML input page"
74-
return HTTP.Response(400, cpu_profile_error_message())
100+
return HTTP.Response(400, profile_error_message())
75101
end
76102
n = convert(Int, parse(Float64, get(qp, "n", default_n())))
77103
delay = parse(Float64, get(qp, "delay", default_delay()))
78104
duration = parse(Float64, get(qp, "duration", default_duration()))
79105
with_pprof = parse(Bool, get(qp, "pprof", default_pprof()))
80-
return handle_cpu_profile(n, delay, duration, with_pprof)
106+
return handle_profile(type, n, delay, duration, with_pprof)
81107
end
82108

83-
function cpu_profile_start_endpoint(req::HTTP.Request)
109+
function profile_start_endpoint(type, req::HTTP.Request)
84110
uri = HTTP.URI(req.target)
85111
qp = HTTP.queryparams(uri)
86112
n = convert(Int, parse(Float64, get(qp, "n", default_n())))
87113
delay = parse(Float64, get(qp, "delay", default_delay()))
88-
return handle_cpu_profile_start(n, delay)
114+
return handle_profile_start(type, n, delay)
89115
end
90116

91-
function cpu_profile_stop_endpoint(req::HTTP.Request)
92-
Profile.stop_timer()
93-
@info "Stopping CPU Profiling from ProfileEndpoints"
117+
function profile_stop_endpoint(type, req::HTTP.Request)
94118
uri = HTTP.URI(req.target)
95119
qp = HTTP.queryparams(uri)
96120
with_pprof = parse(Bool, get(qp, "pprof", default_pprof()))
97-
return handle_cpu_profile_stop(with_pprof)
121+
return handle_profile_stop(with_pprof)
98122
end
99123

100-
function handle_cpu_profile(n, delay, duration, with_pprof, stage_path = nothing)
124+
function handle_profile(type, n, delay, duration, with_pprof, stage_path = nothing)
101125
# Run the profile
102-
return _do_cpu_profile(n, delay, duration, with_pprof, stage_path)
126+
return _do_profile(type, n, delay, duration, with_pprof, stage_path)
103127
end
104128

105-
function _do_cpu_profile(n, delay, duration, with_pprof, stage_path = nothing)
106-
@info "Starting CPU Profiling from ProfileEndpoints with configuration:" n delay duration
129+
function _do_profile(type::TaskProfileType, n, delay, duration, with_pprof, stage_path = nothing)
130+
@info "Starting $type Profiling from ProfileEndpoints with configuration:" n delay duration
107131
Profile.clear()
108132
Profile.init(n, delay)
109-
Profile.@profile sleep(duration)
133+
if type == CPU_PROFILE
134+
Profile.@profile sleep(duration)
135+
elseif type == WALL_PROFILE
136+
@static if isdefined(Profile, Symbol("@profile_walltime"))
137+
Profile.@profile_walltime sleep(duration)
138+
else
139+
return HTTP.Response(501, "You must use a build of Julia (1.12+) that supports walltime profiles.")
140+
end
141+
end
110142
if stage_path === nothing
111143
# Defer the potentially expensive profile symbolication to a non-interactive thread
112-
return fetch(Threads.@spawn _cpu_profile_get_response(with_pprof=$with_pprof))
144+
return fetch(Threads.@spawn _profile_get_response(with_pprof=$with_pprof))
113145
end
114146
path = tempname(stage_path; cleanup=false)
115147
# Defer the potentially expensive profile symbolication to a non-interactive thread
116-
return fetch(Threads.@spawn _cpu_profile_get_response_and_write_to_file($path; with_pprof=$with_pprof))
148+
return fetch(Threads.@spawn _profile_get_response_and_write_to_file($path; with_pprof=$with_pprof))
117149
end
118150

119-
function handle_cpu_profile_start(n, delay)
151+
function handle_profile_start(type, n, delay)
120152
# Run the profile
121-
return _start_cpu_profile(n, delay)
122-
end
123-
124-
function _start_cpu_profile(n, delay)
125-
@info "Starting CPU Profiling from ProfileEndpoints with configuration:" n delay
126-
resp = HTTP.Response(200, "CPU profiling started.")
153+
@info "Starting $type Profiling from ProfileEndpoints with configuration:" n delay
154+
resp = HTTP.Response(200, "$type profiling started.")
127155
Profile.clear()
128156
Profile.init(n, delay)
129-
Profile.start_timer()
157+
if type == CPU_PROFILE
158+
Profile.start_timer()
159+
elseif type == WALL_PROFILE
160+
@static if isdefined(Profile, Symbol("@profile_walltime"))
161+
Profile.start_timer(true)
162+
else
163+
return HTTP.Response(501, "You must use a build of Julia (1.12+) that supports walltime profiles.")
164+
end
165+
end
130166
return resp
131167
end
132168

133-
function handle_cpu_profile_stop(with_pprof, stage_path = nothing)
169+
function handle_profile_stop(with_pprof, stage_path = nothing)
170+
@info "Stopping Profiling from ProfileEndpoints"
171+
Profile.stop_timer()
134172
if stage_path === nothing
135173
# Defer the potentially expensive profile symbolication to a non-interactive thread
136-
return fetch(Threads.@spawn _cpu_profile_get_response(with_pprof=$with_pprof))
174+
return fetch(Threads.@spawn _profile_get_response(with_pprof=$with_pprof))
137175
end
138176
path = tempname(stage_path; cleanup=false)
139177
# Defer the potentially expensive profile symbolication to a non-interactive thread
140-
return fetch(Threads.@spawn _cpu_profile_get_response_and_write_to_file($path; with_pprof=$with_pprof))
178+
return fetch(Threads.@spawn _profile_get_response_and_write_to_file($path; with_pprof=$with_pprof))
141179
end
142180

143-
function _cpu_profile_get_response_and_write_to_file(filename; with_pprof::Bool)
181+
function _profile_get_response_and_write_to_file(filename; with_pprof::Bool)
144182
if with_pprof
145183
PProf.pprof(out=filename, web=false)
146184
filename = "$filename.pb.gz"
@@ -157,7 +195,7 @@ function _cpu_profile_get_response_and_write_to_file(filename; with_pprof::Bool)
157195
end
158196
end
159197

160-
function _cpu_profile_get_response(;with_pprof::Bool)
198+
function _profile_get_response(;with_pprof::Bool)
161199
if with_pprof
162200
prof_name = tempname(;cleanup=false)
163201
PProf.pprof(out=prof_name, web=false)
@@ -189,7 +227,7 @@ function handle_heap_snapshot(all_one, stage_path = nothing)
189227
return HTTP.Response(501, "You must use a build of Julia (1.9+) that supports heap snapshots.")
190228
end
191229

192-
else
230+
else # isdefined
193231

194232
function heap_snapshot_endpoint(req::HTTP.Request)
195233
uri = HTTP.URI(req.target)
@@ -402,21 +440,25 @@ function debug_profile_endpoint_with_stage_path(stage_path = nothing)
402440
profile_dir = subdir
403441
end
404442
profile_type = body["profile_type"]
405-
if profile_type == "cpu_profile"
406-
return handle_cpu_profile(
443+
if profile_type == "cpu_profile" || profile_type == "wall_profile"
444+
type = profile_type == "cpu_profile" ? CPU_PROFILE : WALL_PROFILE
445+
return handle_profile(
446+
type,
407447
convert(Int, parse(Float64, get(body, "n", default_n()))),
408448
parse(Float64, get(body, "delay", default_delay())),
409449
parse(Float64, get(body, "duration", default_duration())),
410450
parse(Bool, get(body, "pprof", default_pprof())),
411451
profile_dir
412452
)
413-
elseif profile_type == "cpu_profile_start"
414-
return handle_cpu_profile_start(
453+
elseif profile_type == "cpu_profile_start" || profile_type == "wall_profile_start"
454+
type = profile_type == "cpu_profile_start" ? CPU_PROFILE : WALL_PROFILE
455+
return handle_profile_start(
456+
type,
415457
convert(Int, parse(Float64, get(body, "n", default_n()))),
416458
parse(Float64, get(body, "delay", default_delay()))
417459
)
418-
elseif profile_type == "cpu_profile_stop"
419-
return handle_cpu_profile_stop(
460+
elseif profile_type == "cpu_profile_stop" || profile_type == "wall_profile_stop"
461+
return handle_profile_stop(
420462
parse(Bool, get(body, "pprof", default_pprof())),
421463
profile_dir
422464
)
@@ -455,6 +497,9 @@ function register_endpoints(router; stage_path = nothing)
455497
HTTP.register!(router, "/profile", cpu_profile_endpoint)
456498
HTTP.register!(router, "/profile_start", cpu_profile_start_endpoint)
457499
HTTP.register!(router, "/profile_stop", cpu_profile_stop_endpoint)
500+
HTTP.register!(router, "/profile_wall", wall_profile_endpoint)
501+
HTTP.register!(router, "/profile_wall_start", wall_profile_start_endpoint)
502+
HTTP.register!(router, "/profile_wall_stop", wall_profile_stop_endpoint)
458503
HTTP.register!(router, "/heap_snapshot", heap_snapshot_endpoint)
459504
HTTP.register!(router, "/allocs_profile", allocations_profile_endpoint)
460505
HTTP.register!(router, "/allocs_profile_start", allocations_start_endpoint)

‎src/precompile.jl

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,16 @@ precompile(serve_profiling_server, ()) || error("precompilation of package funct
33
precompile(cpu_profile_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
44
precompile(cpu_profile_start_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
55
precompile(cpu_profile_stop_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
6-
precompile(_do_cpu_profile, (Int,Float64,Float64,Bool)) || error("precompilation of package functions is not supposed to fail")
7-
precompile(_start_cpu_profile, (Int,Float64,)) || error("precompilation of package functions is not supposed to fail")
6+
precompile(wall_profile_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
7+
precompile(wall_profile_start_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
8+
precompile(wall_profile_stop_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
9+
10+
precompile(debug_profile_endpoint_with_stage_path, (String,)) || error("precompilation of package functions is not supposed to fail")
11+
precompile(debug_profile_endpoint_with_stage_path, (Nothing,)) || error("precompilation of package functions is not supposed to fail")
12+
debug_profile_endpoint_str = debug_profile_endpoint_with_stage_path("stage")
13+
debug_profile_endpoint_nothing = debug_profile_endpoint_with_stage_path("nothing")
14+
precompile(debug_profile_endpoint_str, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
15+
precompile(debug_profile_endpoint_nothing, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
816

917
precompile(heap_snapshot_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
1018

0 commit comments

Comments
 (0)