Skip to content

Commit 3586e72

Browse files
AYadrovAYadrov
committed
new plots based on baseline precisiongit add time.rkt!
1 parent afcfa27 commit 3586e72

File tree

3 files changed

+167
-71
lines changed

3 files changed

+167
-71
lines changed

‎infra/point_graph.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ def plot_points_graph(outcomes, ax):
113113

114114
def load_outcomes(path):
115115
outcomes = json.load(open(path, "r"))["outcomes"]
116-
outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'tool_name', 'number_of_points'])
116+
outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'baseline_precision', 'tool_name', 'number_of_points'])
117117
return outcomes
118118

119119
parser = argparse.ArgumentParser(prog='point_graph.py', description='Script outputs plots for a Herbie run')

‎infra/ratio_plot.py‎

Lines changed: 73 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,51 +4,109 @@
44
import pandas as pd
55
import matplotlib.pyplot as plt
66

7-
def plot_speed_graph(outcomes, ax):
8-
baseline_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-baseline") & (outcomes['rival_iter'] > 73)]
9-
rival_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-rival") & (outcomes['rival_iter'] > 73)]
10-
sollya_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-sollya") & (outcomes['rival_iter'] > 73)]
7+
def plot_speed_graph_rival_iter(outcomes, args):
8+
# Create figure
9+
fig, ax = plt.subplots(figsize=(4, 3.5))
10+
fig.tight_layout(pad=2.0)
11+
12+
# Drop precision column and sum up based on iteration
13+
outcomes = outcomes.drop(['baseline_precision'], axis=1)
14+
outcomes = outcomes.groupby(['rival_iter', 'tool_name'], as_index=False).sum()
15+
16+
# Select appropriate tools
17+
baseline_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-baseline")] # & (outcomes['baseline_precision'] > 73)
18+
rival_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-rival")]
19+
sollya_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-sollya")]
1120

21+
# Some weird functions that creates speed per millisecond for each tool
1222
def add_values(row):
1323
return int(row['rival_iter']), (row['number_of_points'] / row['time']) * 1000
14-
1524
def tool_cmp2speed(x):
1625
return x.sort_values(by=['rival_iter']).apply(add_values, axis=1, result_type='expand')
1726

27+
# Sollya timings considered are as base since we are doing speed ratio comparison
1828
base = np.array(tool_cmp2speed(sollya_cmp)[1])
1929

30+
# Plot Rival
2031
ax.plot(tool_cmp2speed(rival_cmp)[0], np.array(tool_cmp2speed(rival_cmp)[1])/base, '.-', linewidth=2.0, color='r', label='reval')
32+
# Plot Baseline
2133
ax.plot(tool_cmp2speed(baseline_cmp)[0], np.array(tool_cmp2speed(baseline_cmp)[1])/base, '--', linewidth=2.0, color='g',
2234
label='baseline')
35+
# Plot Sollya
2336
ax.plot(tool_cmp2speed(sollya_cmp)[0], np.array(tool_cmp2speed(sollya_cmp)[1])/base, '-', linewidth=2.0, color='b',
2437
label='sollya')
2538

39+
ax.legend()
40+
ax.set_xlabel("Difficulty")
41+
ax.set_ylabel("Ratio")
42+
ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3)
43+
plt.savefig(args.path + "/ratio_plot_iter.png", format="png")
44+
plt.savefig(args.path + "/ratio_plot_iter.pdf", format="pdf")
45+
46+
# Latex stuff
2647
print("\\newcommand{\RivalAvgSpeedupOverSollya}{" + str(round(tool_cmp2speed(rival_cmp)[1].sum()/np.array(tool_cmp2speed(sollya_cmp)[1]).sum(), 2)) + "\\xspace}")
2748
print("\\newcommand{\RivalAvgSpeedupOverBaseline}{" + str(
2849
round(tool_cmp2speed(rival_cmp)[1].sum() / np.array(tool_cmp2speed(baseline_cmp)[1]).sum(), 2)) + "\\xspace}")
29-
3050
print("\\newcommand{\RivalMaxSpeedupOverSollya}{" + str(round(np.array(tool_cmp2speed(rival_cmp)[1])[-1]/np.array(tool_cmp2speed(sollya_cmp)[1])[-1], 2)) + "\\xspace}")
3151
print("\\newcommand{\RivalMaxSpeedupOverBaseline}{" + str(round(np.array(tool_cmp2speed(rival_cmp)[1])[-1]/np.array(tool_cmp2speed(baseline_cmp)[1])[-1], 2)) + "\\xspace}")
3252

53+
def plot_speed_graph_baseline_precision(outcomes, args):
54+
# Create figure
55+
fig, ax = plt.subplots(figsize=(4, 3.5))
56+
fig.tight_layout(pad=2.0)
57+
58+
# Drop precision column and sum up based on iteration
59+
outcomes = outcomes.drop(['rival_iter'], axis=1)
60+
outcomes = outcomes.groupby(['baseline_precision', 'tool_name'], as_index=False).sum()
61+
62+
# Select appropriate tools
63+
baseline_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-baseline") & (outcomes['baseline_precision'] > 73)]
64+
rival_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-rival") & (outcomes['baseline_precision'] > 73)]
65+
sollya_cmp = outcomes.loc[(outcomes['tool_name'] == "valid-sollya") & (outcomes['baseline_precision'] > 73)]
66+
67+
# Some weird functions that creates speed per millisecond for each tool
68+
def add_values(row):
69+
return int(row['baseline_precision']), (row['number_of_points'] / row['time']) * 1000
70+
def tool_cmp2speed(x):
71+
return x.sort_values(by=['baseline_precision']).apply(add_values, axis=1, result_type='expand')
72+
73+
# Sollya timings considered are as base since we are doing speed ratio comparison
74+
base = np.array(tool_cmp2speed(sollya_cmp)[1])
75+
76+
# Plot Rival
77+
ax.plot(tool_cmp2speed(rival_cmp)[0], np.array(tool_cmp2speed(rival_cmp)[1])/base, '.-', linewidth=2.0, color='r', label='reval')
78+
# Plot Baseline
79+
ax.plot(tool_cmp2speed(baseline_cmp)[0], np.array(tool_cmp2speed(baseline_cmp)[1])/base, '--', linewidth=2.0, color='g',
80+
label='baseline')
81+
# Plot Sollya
82+
ax.plot(tool_cmp2speed(sollya_cmp)[0], np.array(tool_cmp2speed(sollya_cmp)[1])/base, '-', linewidth=2.0, color='b',
83+
label='sollya')
84+
3385
ax.legend()
34-
ax.set_xlabel("Difficulty")
86+
ax.set_xlabel("Truth uniform precision")
3587
ax.set_ylabel("Ratio")
3688
ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3)
37-
89+
plt.savefig(args.path + "/ratio_plot_precision.png", format="png")
90+
plt.savefig(args.path + "/ratio_plot_precision.pdf", format="pdf")
91+
92+
# Latex stuff
93+
print("\\newcommand{\RivalAvgSpeedupOverSollya}{" + str(round(tool_cmp2speed(rival_cmp)[1].sum()/np.array(tool_cmp2speed(sollya_cmp)[1]).sum(), 2)) + "\\xspace}")
94+
print("\\newcommand{\RivalAvgSpeedupOverBaseline}{" + str(
95+
round(tool_cmp2speed(rival_cmp)[1].sum() / np.array(tool_cmp2speed(baseline_cmp)[1]).sum(), 2)) + "\\xspace}")
96+
print("\\newcommand{\RivalMaxSpeedupOverSollya}{" + str(round(np.array(tool_cmp2speed(rival_cmp)[1])[-1]/np.array(tool_cmp2speed(sollya_cmp)[1])[-1], 2)) + "\\xspace}")
97+
print("\\newcommand{\RivalMaxSpeedupOverBaseline}{" + str(round(np.array(tool_cmp2speed(rival_cmp)[1])[-1]/np.array(tool_cmp2speed(baseline_cmp)[1])[-1], 2)) + "\\xspace}")
98+
3899
def load_outcomes(path):
39100
outcomes = json.load(open(path, "r"))["outcomes"]
40-
outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'tool_name', 'number_of_points'])
101+
outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'baseline_precision', 'tool_name', 'number_of_points'])
41102
return outcomes
42103

43-
parser = argparse.ArgumentParser(prog='histograms.py', description='Script outputs mixed precision histograms for a Herbie run')
104+
parser = argparse.ArgumentParser(prog='ratio_plot.py', description='Script outputs ratio plots')
44105
parser.add_argument('-t', '--timeline', dest='timeline', default="report/timeline.json")
45106
parser.add_argument('-o', '--output-path', dest='path', default="report")
46107
args = parser.parse_args()
47108

48109
outcomes = load_outcomes(args.timeline)
110+
plot_speed_graph_rival_iter(outcomes, args)
111+
plot_speed_graph_baseline_precision(outcomes, args)
49112

50-
fig, ax = plt.subplots(figsize=(4, 3.5))
51-
fig.tight_layout(pad=2.0)
52-
plot_speed_graph(outcomes, ax)
53-
plt.savefig(args.path + "/ratio_plot.png", format="png")
54-
plt.savefig(args.path + "/ratio_plot.pdf", format="pdf")

‎time.rkt‎

Lines changed: 93 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@
6969
(for/list ([pt (in-list (hash-ref rec 'points))])
7070
; --------------------------- Rival execution -------------------------------------------------
7171
(define rival-start-apply (current-inexact-milliseconds))
72-
7372
(match-define (list rival-status rival-exs)
7473
(parameterize ([*rival-max-precision* 32256])
7574
(with-handlers ([exn:rival:invalid? (λ (e) (list 'invalid #f))]
@@ -156,7 +155,8 @@
156155
sollya-status
157156
sollya-apply-time
158157
sollya-exs
159-
baseline-precision)))
158+
baseline-precision
159+
rival-iter)))
160160

161161
; Count differences where baseline is better than rival
162162
(define rival-baseline-difference
@@ -209,11 +209,11 @@
209209
(define (timeline-push! timeline key args*)
210210
(match key
211211
['outcomes
212-
(match-define (list status iter time*) args*)
212+
(match-define (list status iter precision time*) args*)
213213
(define outcomes-hash (hash-ref timeline key))
214214
(match-define (list time num-points)
215-
(hash-ref outcomes-hash (list status iter) (λ () (list 0 0))))
216-
(hash-set! outcomes-hash (list status iter) (list (+ time time*) (+ num-points 1)))]
215+
(hash-ref outcomes-hash (list status iter precision) (λ () (list 0 0))))
216+
(hash-set! outcomes-hash (list status iter precision) (list (+ time time*) (+ num-points 1)))]
217217
[(or 'mixsample-rival-valid
218218
'mixsample-rival-all
219219
'mixsample-baseline-valid
@@ -227,7 +227,7 @@
227227
(define (timeline->jsexpr timeline)
228228
(hash 'outcomes
229229
(for/list ([(key value) (in-hash (hash-ref timeline 'outcomes))])
230-
(list (first value) (second key) (first key) (second value)))
230+
(list (first value) (second key) (third key) (first key) (second value)))
231231
'mixsample-rival-valid
232232
(for/list ([(key value) (in-hash (hash-ref timeline 'mixsample-rival-valid))])
233233
(list value (car key) (second key)))
@@ -400,7 +400,8 @@
400400
(html-end-table html-port))
401401

402402
(when expression-table
403-
(html-add-plot html-port "ratio_plot.png")
403+
(html-add-plot html-port "ratio_plot_iter.png")
404+
(html-add-plot html-port "ratio_plot_precision.png")
404405
(html-add-plot html-port "point_graph.png")
405406
(html-add-histogram html-port "histogram_valid.png")
406407
(html-add-histogram html-port "histogram_all.png"))
@@ -457,6 +458,7 @@
457458
sollya-status
458459
sollya-time
459460
sollya-exs
461+
baseline-precision
460462
rival-iter)
461463

462464
(define (status-subbucketing status exs)
@@ -474,89 +476,125 @@
474476
[(and (equal? 'valid sollya-status)
475477
(equal? 'valid baseline-status)
476478
(equal? rival-status 'valid))
477-
(timeline-push! timeline 'outcomes (list "valid-sollya" rival-iter sollya-time))
478-
(timeline-push! timeline 'outcomes (list "valid-baseline" rival-iter baseline-time))
479-
(timeline-push! timeline 'outcomes (list "valid-rival" rival-iter rival-time))
479+
(timeline-push! timeline
480+
'outcomes
481+
(list "valid-sollya" rival-iter baseline-precision sollya-time))
482+
(timeline-push! timeline
483+
'outcomes
484+
(list "valid-baseline" rival-iter baseline-precision baseline-time))
485+
(timeline-push! timeline
486+
'outcomes
487+
(list "valid-rival" rival-iter baseline-precision rival-time))
480488
(if (fl= rival-exs sollya-exs)
481-
(timeline-push! timeline 'outcomes (list "sollya-correct-rounding" 0 0))
489+
(timeline-push! timeline 'outcomes (list "sollya-correct-rounding" 0 0 0))
482490
(if (equal? (flonums-between rival-exs sollya-exs) 1)
483-
(timeline-push! timeline 'outcomes (list "sollya-faithful-rounding" 0 0))
484-
(timeline-push! timeline 'outcomes (list "sollya-off-results" 0 0))))]
491+
(timeline-push! timeline 'outcomes (list "sollya-faithful-rounding" 0 0 0))
492+
(timeline-push! timeline 'outcomes (list "sollya-off-results" 0 0 0))))]
485493

486494
; Baseline and Rival have succeeded
487495
[(and (equal? 'valid baseline-status) (equal? rival-status 'valid))
488-
(timeline-push!
489-
timeline
490-
'outcomes
491-
(list (status-subbucketing "valid-rival+baseline" rival-exs) rival-iter rival-time))]
496+
(timeline-push! timeline
497+
'outcomes
498+
(list (status-subbucketing "valid-rival+baseline" rival-exs)
499+
rival-iter
500+
baseline-precision
501+
rival-time))]
492502

493503
; Baseline and Sollya have succeeded
494504
[(and (equal? 'valid sollya-status) (equal? 'valid baseline-status))
495-
(timeline-push!
496-
timeline
497-
'outcomes
498-
(list (status-subbucketing "valid-sollya+baseline" baseline-exs) rival-iter sollya-time))]
505+
(timeline-push! timeline
506+
'outcomes
507+
(list (status-subbucketing "valid-sollya+baseline" baseline-exs)
508+
rival-iter
509+
baseline-precision
510+
sollya-time))]
499511

500512
; Sollya and Rival have succeeded
501513
[(and (equal? 'valid sollya-status) (equal? rival-status 'valid))
502-
(timeline-push!
503-
timeline
504-
'outcomes
505-
(list (status-subbucketing "valid-rival+sollya" rival-exs) rival-iter rival-time))]
514+
(timeline-push! timeline
515+
'outcomes
516+
(list (status-subbucketing "valid-rival+sollya" rival-exs)
517+
rival-iter
518+
baseline-precision
519+
rival-time))]
506520

507521
; Only Rival has succeeded
508522
[(equal? rival-status 'valid)
509-
(timeline-push!
510-
timeline
511-
'outcomes
512-
(list (status-subbucketing "valid-rival-only" rival-exs) rival-iter rival-time))]
523+
(timeline-push! timeline
524+
'outcomes
525+
(list (status-subbucketing "valid-rival-only" rival-exs)
526+
rival-iter
527+
baseline-precision
528+
rival-time))]
513529

514530
; Only Sollya has succeeded
515531
[(equal? 'valid sollya-status)
516-
(timeline-push!
517-
timeline
518-
'outcomes
519-
(list (status-subbucketing "valid-sollya-only" sollya-exs) rival-iter sollya-time))]
532+
(timeline-push! timeline
533+
'outcomes
534+
(list (status-subbucketing "valid-sollya-only" sollya-exs)
535+
rival-iter
536+
baseline-precision
537+
sollya-time))]
520538

521539
; Only Baseline has succeeded
522540
[(equal? 'valid baseline-status)
523-
(timeline-push!
524-
timeline
525-
'outcomes
526-
(list (status-subbucketing "valid-baseline-only" baseline-exs) rival-iter baseline-time))]
541+
(timeline-push! timeline
542+
'outcomes
543+
(list (status-subbucketing "valid-baseline-only" baseline-exs)
544+
rival-iter
545+
baseline-precision
546+
baseline-time))]
527547

528548
; timeout at all the tools
529549
[else
530-
(timeline-push! timeline 'outcomes (list "exit-baseline" rival-iter baseline-time))
531-
(timeline-push! timeline 'outcomes (list "exit-sollya" rival-iter sollya-time))
532-
(timeline-push! timeline 'outcomes (list "exit-rival" rival-iter rival-time))])]
550+
(timeline-push! timeline
551+
'outcomes
552+
(list "exit-baseline" rival-iter baseline-precision baseline-time))
553+
(timeline-push! timeline
554+
'outcomes
555+
(list "exit-sollya" rival-iter baseline-precision sollya-time))
556+
(timeline-push! timeline
557+
'outcomes
558+
(list "exit-rival" rival-iter baseline-precision rival-time))])]
533559

534560
; Rival has exited
535561
[(equal? rival-status 'unsamplable)
536562
(cond
537563
; Sollya and Baseline have succeeded
538564
[(and (equal? 'valid sollya-status) (equal? 'valid baseline-status))
539-
(timeline-push!
540-
timeline
541-
'outcomes
542-
(list (status-subbucketing "valid-sollya+baseline" baseline-exs) rival-iter sollya-time))]
565+
(timeline-push! timeline
566+
'outcomes
567+
(list (status-subbucketing "valid-sollya+baseline" baseline-exs)
568+
rival-iter
569+
baseline-precision
570+
sollya-time))]
543571

544572
; Only Sollya has succeeded
545573
[(equal? 'valid sollya-status)
546-
(timeline-push!
547-
timeline
548-
'outcomes
549-
(list (status-subbucketing "valid-sollya-only" sollya-exs) rival-iter sollya-time))]
574+
(timeline-push! timeline
575+
'outcomes
576+
(list (status-subbucketing "valid-sollya-only" sollya-exs)
577+
rival-iter
578+
baseline-precision
579+
sollya-time))]
550580

551581
; Only Baseline has succeeded
552582
[(equal? 'valid baseline-status)
553-
(timeline-push!
554-
timeline
555-
'outcomes
556-
(list (status-subbucketing "valid-baseline-only" baseline-exs) rival-iter baseline-time))]
583+
(timeline-push! timeline
584+
'outcomes
585+
(list (status-subbucketing "valid-baseline-only" baseline-exs)
586+
rival-iter
587+
baseline-precision
588+
baseline-time))]
557589

558590
; Points that every tools fail to evaluate when the precision is unreacheble
559591
[else
560-
(timeline-push! timeline 'outcomes (list "exit-baseline" rival-iter baseline-time))
561-
(timeline-push! timeline 'outcomes (list "exit-sollya" rival-iter sollya-time))
562-
(timeline-push! timeline 'outcomes (list "exit-rival" rival-iter rival-time))])]))
592+
(timeline-push! timeline
593+
'outcomes
594+
(list "exit-baseline" rival-iter baseline-precision baseline-time))
595+
(timeline-push! timeline
596+
'outcomes
597+
(list "exit-sollya" rival-iter baseline-precision sollya-time))
598+
(timeline-push! timeline
599+
'outcomes
600+
(list "exit-rival" rival-iter baseline-precision rival-time))])]))

0 commit comments

Comments
 (0)