Skip to content

Commit 88c6a28

Browse files
rodrigo-britorodrigo-brito
authored
feat: include confidence interval / profit factor (#294)
1 parent 31c3f11 commit 88c6a28

File tree

9 files changed

+199
-14
lines changed

9 files changed

+199
-14
lines changed

‎exchange/pairs.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎exchange/paperwallet.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ func (p *PaperWallet) Summary() {
181181
volume float64
182182
)
183183

184-
fmt.Println("-- FINAL WALLET --")
184+
fmt.Println("----- FINAL WALLET -----")
185185
for pair := range p.lastCandle {
186186
asset, quote := SplitAssetQuote(pair)
187187
assetInfo, ok := p.assets[asset]

‎go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ require (
1919
github.com/urfave/cli/v2 v2.25.7
2020
github.com/vektra/mockery/v2 v2.36.0
2121
github.com/xhit/go-str2duration/v2 v2.1.0
22-
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
22+
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
23+
gonum.org/v1/gonum v0.13.0
2324
gopkg.in/tucnak/telebot.v2 v2.5.0
2425
gorm.io/gorm v1.25.5
2526
)
@@ -76,7 +77,7 @@ require (
7677
golang.org/x/mod v0.9.0 // indirect
7778
golang.org/x/sys v0.7.0 // indirect
7879
golang.org/x/term v0.6.0 // indirect
79-
golang.org/x/text v0.7.0 // indirect
80+
golang.org/x/text v0.8.0 // indirect
8081
golang.org/x/tools v0.7.0 // indirect
8182
gopkg.in/ini.v1 v1.67.0 // indirect
8283
gopkg.in/yaml.v2 v2.4.0 // indirect

‎go.sum‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -316,8 +316,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
316316
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
317317
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
318318
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
319-
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
320-
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
319+
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
320+
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
321321
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
322322
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
323323
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -445,8 +445,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
445445
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
446446
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
447447
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
448-
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
449-
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
448+
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
449+
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
450450
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
451451
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
452452
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -503,6 +503,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
503503
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
504504
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
505505
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
506+
gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM=
507+
gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU=
506508
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
507509
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
508510
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

‎ninjabot.go‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/rodrigo-brito/ninjabot/storage"
1818
"github.com/rodrigo-brito/ninjabot/strategy"
1919
"github.com/rodrigo-brito/ninjabot/tools/log"
20+
"github.com/rodrigo-brito/ninjabot/tools/metrics"
2021

2122
"github.com/olekukonko/tablewriter"
2223
"github.com/schollz/progressbar/v3"
@@ -191,20 +192,23 @@ func (n *NinjaBot) Summary() {
191192

192193
buffer := bytes.NewBuffer(nil)
193194
table := tablewriter.NewWriter(buffer)
194-
table.SetHeader([]string{"Pair", "Trades", "Win", "Loss", "% Win", "Payoff", "SQN", "Profit", "Volume"})
195+
table.SetHeader([]string{"Pair", "Trades", "Win", "Loss", "% Win", "Payoff", "Pr Fact.", "SQN", "Profit", "Volume"})
195196
table.SetFooterAlignment(tablewriter.ALIGN_RIGHT)
196197
avgPayoff := 0.0
198+
avgProfitFactor := 0.0
197199

198200
returns := make([]float64, 0)
199201
for _, summary := range n.orderController.Results {
200202
avgPayoff += summary.Payoff() * float64(len(summary.Win())+len(summary.Lose()))
203+
avgProfitFactor += summary.ProfitFactor() * float64(len(summary.Win())+len(summary.Lose()))
201204
table.Append([]string{
202205
summary.Pair,
203206
strconv.Itoa(len(summary.Win()) + len(summary.Lose())),
204207
strconv.Itoa(len(summary.Win())),
205208
strconv.Itoa(len(summary.Lose())),
206209
fmt.Sprintf("%.1f %%", float64(len(summary.Win()))/float64(len(summary.Win())+len(summary.Lose()))*100),
207210
fmt.Sprintf("%.3f", summary.Payoff()),
211+
fmt.Sprintf("%.3f", summary.ProfitFactor()),
208212
fmt.Sprintf("%.1f", summary.SQN()),
209213
fmt.Sprintf("%.2f", summary.Profit()),
210214
fmt.Sprintf("%.2f", summary.Volume),
@@ -226,6 +230,7 @@ func (n *NinjaBot) Summary() {
226230
strconv.Itoa(loses),
227231
fmt.Sprintf("%.1f %%", float64(wins)/float64(wins+loses)*100),
228232
fmt.Sprintf("%.3f", avgPayoff/float64(wins+loses)),
233+
fmt.Sprintf("%.3f", avgProfitFactor/float64(wins+loses)),
229234
fmt.Sprintf("%.1f", sqn/float64(len(n.orderController.Results))),
230235
fmt.Sprintf("%.2f", total),
231236
fmt.Sprintf("%.2f", volume),
@@ -240,17 +245,44 @@ func (n *NinjaBot) Summary() {
240245
returnsPercent = append(returnsPercent, p*100)
241246
totalReturn += p
242247
}
243-
fmt.Printf("AVG Return: %.2f%%\n", totalReturn/float64(len(returns))*100)
244-
hist := histogram.Hist(20, returnsPercent)
248+
hist := histogram.Hist(15, returnsPercent)
245249
histogram.Fprint(os.Stdout, hist, histogram.Linear(10))
246250
fmt.Println()
247251

252+
fmt.Println("------ CONFIDENCE INTERVAL (95%) -------")
253+
for pair, summary := range n.orderController.Results {
254+
fmt.Printf("| %s |\n", pair)
255+
returns := append(summary.WinPercent(), summary.LosePercent()...)
256+
returnsInterval := metrics.Bootstrap(returns, metrics.Mean, 10000, 0.95)
257+
payoffInterval := metrics.Bootstrap(returns, metrics.Payoff, 10000, 0.95)
258+
profitFactorInterval := metrics.Bootstrap(returns, metrics.ProfitFactor, 10000, 0.95)
259+
260+
fmt.Printf("RETURN: %.2f%% (%.2f%% ~ %.2f%%)\n",
261+
returnsInterval.Mean*100, returnsInterval.Lower*100, returnsInterval.Upper*100)
262+
fmt.Printf("PAYOFF: %.2f (%.2f ~ %.2f)\n",
263+
payoffInterval.Mean, payoffInterval.Lower, payoffInterval.Upper)
264+
fmt.Printf("PROF.FACTOR: %.2f (%.2f ~ %.2f)\n",
265+
profitFactorInterval.Mean, profitFactorInterval.Lower, profitFactorInterval.Upper)
266+
}
267+
268+
fmt.Println()
269+
248270
if n.paperWallet != nil {
249271
n.paperWallet.Summary()
250272
}
251273

252274
}
253275

276+
func (n NinjaBot) SaveReturns(outputDir string) error {
277+
for _, summary := range n.orderController.Results {
278+
outputFile := fmt.Sprintf("%s/%s.csv", outputDir, summary.Pair)
279+
if err := summary.SaveReturns(outputFile); err != nil {
280+
return err
281+
}
282+
}
283+
return nil
284+
}
285+
254286
func (n *NinjaBot) onCandle(candle model.Candle) {
255287
n.priorityQueueCandle.Push(candle)
256288
}

‎order/controller.go‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"math"
7+
"os"
78
"strconv"
89
"strings"
910
"sync"
@@ -70,11 +71,11 @@ func (s summary) Payoff() float64 {
7071
avgWin := 0.0
7172
avgLose := 0.0
7273

73-
for _, value := range s.Win() {
74+
for _, value := range s.WinPercent() {
7475
avgWin += value
7576
}
7677

77-
for _, value := range s.Lose() {
78+
for _, value := range s.LosePercent() {
7879
avgLose += value
7980
}
8081

@@ -85,6 +86,22 @@ func (s summary) Payoff() float64 {
8586
return (avgWin / float64(len(s.Win()))) / math.Abs(avgLose/float64(len(s.Lose())))
8687
}
8788

89+
func (s summary) ProfitFactor() float64 {
90+
if len(s.Lose()) == 0 {
91+
return 0
92+
}
93+
profit := 0.0
94+
for _, value := range s.WinPercent() {
95+
profit += value
96+
}
97+
98+
loss := 0.0
99+
for _, value := range s.LosePercent() {
100+
loss += value
101+
}
102+
return profit / math.Abs(loss)
103+
}
104+
88105
func (s summary) WinPercentage() float64 {
89106
if len(s.Win())+len(s.Lose()) == 0 {
90107
return 0
@@ -104,6 +121,7 @@ func (s summary) String() string {
104121
{"Loss", strconv.Itoa(len(s.Lose()))},
105122
{"% Win", fmt.Sprintf("%.1f", s.WinPercentage())},
106123
{"Payoff", fmt.Sprintf("%.1f", s.Payoff()*100)},
124+
{"Pr.Fact", fmt.Sprintf("%.1f", s.Payoff()*100)},
107125
{"Profit", fmt.Sprintf("%.4f %s", s.Profit(), quote)},
108126
{"Volume", fmt.Sprintf("%.4f %s", s.Volume, quote)},
109127
}
@@ -113,6 +131,29 @@ func (s summary) String() string {
113131
return tableString.String()
114132
}
115133

134+
func (s summary) SaveReturns(filename string) error {
135+
file, err := os.Create(filename)
136+
if err != nil {
137+
return err
138+
}
139+
defer file.Close()
140+
141+
for _, value := range s.WinPercent() {
142+
_, err = file.WriteString(fmt.Sprintf("%.4f\n", value))
143+
if err != nil {
144+
return err
145+
}
146+
}
147+
148+
for _, value := range s.LosePercent() {
149+
_, err = file.WriteString(fmt.Sprintf("%.4f\n", value))
150+
if err != nil {
151+
return err
152+
}
153+
}
154+
return nil
155+
}
156+
116157
type Status string
117158

118159
const (
@@ -237,7 +278,7 @@ func (c *Controller) updatePosition(o *model.Order) {
237278

238279
if result != nil {
239280
// TODO: replace by a slice of Result
240-
if result.ProfitPercent > 0 {
281+
if result.ProfitPercent >= 0 {
241282
if result.Side == model.SideTypeBuy {
242283
c.Results[o.Pair].WinLong = append(c.Results[o.Pair].WinLong, result.ProfitValue)
243284
c.Results[o.Pair].WinLongPercent = append(c.Results[o.Pair].WinLongPercent, result.ProfitPercent)

‎tools/metrics/bootstrap.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package metrics
2+
3+
import (
4+
"sort"
5+
6+
"github.com/samber/lo"
7+
"gonum.org/v1/gonum/stat"
8+
)
9+
10+
type BootstrapInterval struct {
11+
Lower float64
12+
Upper float64
13+
StdDev float64
14+
Mean float64
15+
}
16+
17+
// Bootstrap calculates the confidence interval of a sample using the bootstrap method.
18+
func Bootstrap(values []float64, measure func([]float64) float64, sampleSize int,
19+
confidence float64) BootstrapInterval {
20+
21+
var data []float64
22+
for i := 0; i < sampleSize; i++ {
23+
samples := make([]float64, len(values))
24+
for j := 0; j < len(values); j++ {
25+
samples[j] = lo.Sample(values)
26+
}
27+
data = append(data, measure(samples))
28+
}
29+
30+
tail := 1 - confidence
31+
32+
sort.Float64s(data)
33+
mean, stdDev := stat.MeanStdDev(data, nil)
34+
upper := stat.Quantile(1-tail/2, stat.LinInterp, data, nil)
35+
lower := stat.Quantile(tail/2, stat.LinInterp, data, nil)
36+
37+
return BootstrapInterval{
38+
Lower: lower,
39+
Upper: upper,
40+
StdDev: stdDev,
41+
Mean: mean,
42+
}
43+
}

‎tools/metrics/bootstrap_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package metrics
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/require"
7+
"gonum.org/v1/gonum/stat"
8+
)
9+
10+
func TestBootstrap(t *testing.T) {
11+
values := []float64{7, 9, 10, 10, 12, 14, 15, 16, 16, 17, 19, 20, 21, 21, 23}
12+
result := Bootstrap(values, func(samples []float64) float64 {
13+
return stat.Mean(samples, nil)
14+
}, 10000, 0.95)
15+
16+
require.InDelta(t, 15.34, result.Mean, 0.1)
17+
require.InDelta(t, 1.24, result.StdDev, 0.1)
18+
require.InDelta(t, 12.9, result.Lower, 0.1)
19+
require.InDelta(t, 17.7, result.Upper, 0.1)
20+
}

‎tools/metrics/metrics.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package metrics
2+
3+
import (
4+
"math"
5+
6+
"gonum.org/v1/gonum/stat"
7+
)
8+
9+
func Mean(values []float64) float64 {
10+
return stat.Mean(values, nil)
11+
}
12+
13+
func Payoff(values []float64) float64 {
14+
wins := []float64{}
15+
loses := []float64{}
16+
for _, value := range values {
17+
if value >= 0 {
18+
wins = append(wins, value)
19+
} else {
20+
loses = append(loses, value)
21+
}
22+
}
23+
24+
return math.Abs(stat.Mean(wins, nil) / stat.Mean(loses, nil))
25+
}
26+
27+
func ProfitFactor(values []float64) float64 {
28+
var (
29+
wins float64
30+
loses float64
31+
)
32+
33+
for _, value := range values {
34+
if value >= 0 {
35+
wins += value
36+
} else {
37+
loses += value
38+
}
39+
}
40+
41+
if loses == 0 {
42+
return 10
43+
}
44+
45+
return math.Abs(wins / loses)
46+
}

0 commit comments

Comments
 (0)