Skip to content

Commit

Permalink
Split up weight into BoostWeight and Flatness Weight, Flatness is onl…
Browse files Browse the repository at this point in the history 
…y used in Tree, Boost is used everywhere
  • Loading branch information
@thomaskeck
thomaskeck committed Jun 6, 2017
1 parent 59ef6a4 commit 9a38430
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 73 deletions.
6 changes: 3 additions & 3 deletions examples/ugboost.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ def evaluation(label, X_test, y_test, p, p_prior):
if __name__ == '__main__':
# Create some Monte Carlo data using a multidimensional gaussian distribution
# The 0th row of the coveriance matrix describes the correlation to the target variable
mean = [0.5, 0.6, 0.6]
mean = [0.5, 0.4, 0.4]
cov = [[1.0, 0.6, 0.6],
[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0]]

mean2 = [-0.5, -0.6, -0.6]
mean2 = [-0.5, -0.4, -0.4]
cov2 = [[1.0, 0.6, 0.6],
[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0]]
Expand All @@ -92,7 +92,7 @@ def evaluation(label, X_test, y_test, p, p_prior):

evaluation("Random", X_test, y_test, np.random.uniform(size=N_test), p_prior)

for i in [0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0]:
for i in [0.0, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0]:
p = FastBDT.Classifier(flatnessLoss=i, numberOfFlatnessFeatures=1).fit(X=np.c_[X_train[:, 1:], X_train[:, 0]], y=y_train).predict(X_test[:, 1:])
print("Flatness", i)
evaluation("UBoost", X_test, y_test, p, p_prior)
Expand Down
18 changes: 10 additions & 8 deletions include/FastBDT.h.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,14 +518,15 @@ namespace FastBDT {
class EventWeights {

public:
EventWeights(unsigned int nEvents) : weights(nEvents, 1), original_weights(nEvents, 0) { }
EventWeights(unsigned int nEvents) : boost_weights(nEvents, 1), flatness_weights(nEvents, 0), original_weights(nEvents, 0) { }

inline Weight Get(unsigned int iEvent) const { return weights[iEvent] * original_weights[iEvent]; }
inline Weight GetWithoutOriginal(unsigned int iEvent) const { return weights[iEvent]; }
void Set(unsigned int iEvent, const Weight& weight) { weights[iEvent] = weight; }

inline const Weight& GetOriginal(unsigned int iEvent) const { return original_weights[iEvent]; }
void SetOriginal(unsigned int iEvent, const Weight& weight) { original_weights[iEvent] = weight; }
inline Weight Get(unsigned int iEvent) const { return boost_weights[iEvent] * original_weights[iEvent]; }
inline const Weight& GetBoostWeight(unsigned int iEvent) const { return boost_weights[iEvent]; }
inline const Weight& GetFlatnessWeight(unsigned int iEvent) const { return flatness_weights[iEvent]; }
inline const Weight& GetOriginalWeight(unsigned int iEvent) const { return original_weights[iEvent]; }
void SetBoostWeight(unsigned int iEvent, const Weight& weight) { boost_weights[iEvent] = weight; }
void SetFlatnessWeight(unsigned int iEvent, const Weight& weight) { flatness_weights[iEvent] = weight; }
void SetOriginalWeight(unsigned int iEvent, const Weight& weight) { original_weights[iEvent] = weight; }

/**
* Returns the sum of all weights. 0: SignalSum, 1: BckgrdSum, 2: SquareSum
Expand All @@ -534,7 +535,8 @@ namespace FastBDT {
std::vector<Weight> GetSums(unsigned int nSignals) const;

private:
std::vector<Weight> weights;
std::vector<Weight> boost_weights;
std::vector<Weight> flatness_weights;
std::vector<Weight> original_weights;
};

Expand Down
63 changes: 25 additions & 38 deletions src/FastBDT.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ namespace FastBDT {
// Vectorizing FTW!
std::vector<Weight> sums(3,0);
for(unsigned int i = 0; i < nSignals; ++i) {
sums[0] += weights[i] * original_weights[i];
sums[2] += weights[i]*weights[i] * original_weights[i];
sums[0] += boost_weights[i] * original_weights[i];
sums[2] += boost_weights[i]*boost_weights[i] * original_weights[i];
}

for(unsigned int i = nSignals; i < weights.size(); ++i) {
sums[1] += weights[i] * original_weights[i];
sums[2] += weights[i]*weights[i] * original_weights[i];
for(unsigned int i = nSignals; i < original_weights.size(); ++i) {
sums[1] += boost_weights[i] * original_weights[i];
sums[2] += boost_weights[i]*boost_weights[i] * original_weights[i];
}
return sums;

Expand Down Expand Up @@ -89,7 +89,7 @@ namespace FastBDT {
index = nEvents - 1 - nBckgrds;
++nBckgrds;
}
weights.SetOriginal(index, weight);
weights.SetOriginalWeight(index, weight);
values.Set(index, features);

}
Expand Down Expand Up @@ -130,7 +130,7 @@ namespace FastBDT {
const unsigned int index = (flags.Get(iEvent)-nNodes)*nBinSums[nFeatures];
for(unsigned int iFeature = 0; iFeature < nFeatures; ++iFeature ) {
const unsigned int subindex = nBinSums[iFeature] + values.Get(iEvent,iFeature);
bins[index+subindex] += weights.Get(iEvent);
bins[index+subindex] += weights.GetOriginalWeight(iEvent) * (weights.GetBoostWeight(iEvent) + weights.GetFlatnessWeight(iEvent));
}
}

Expand Down Expand Up @@ -183,16 +183,16 @@ namespace FastBDT {
void Node::AddSignalWeight(Weight weight, Weight original_weight) {
if(original_weight == 0)
return;
signal += weight;
square += weight*weight / original_weight;
signal += weight * original_weight;
square += weight*weight * original_weight;
}


void Node::AddBckgrdWeight(Weight weight, Weight original_weight) {
if(original_weight == 0)
return;
bckgrd += weight;
square += weight*weight / original_weight;
bckgrd += weight * original_weight;
square += weight*weight * original_weight;
}

void Node::SetWeights(std::vector<Weight> weights) {
Expand Down Expand Up @@ -324,13 +324,13 @@ namespace FastBDT {
for(unsigned int iEvent = 0; iEvent < sample.GetNSignals(); ++iEvent) {
const int flag = flags.Get(iEvent);
if( flag >= static_cast<int>(nNodes) ) {
nodes[flag-1].AddSignalWeight( weights.Get(iEvent), weights.GetOriginal(iEvent) );
nodes[flag-1].AddSignalWeight( weights.GetBoostWeight(iEvent), weights.GetOriginalWeight(iEvent) );
}
}
for(unsigned int iEvent = sample.GetNSignals(); iEvent < sample.GetNEvents(); ++iEvent) {
const int flag = flags.Get(iEvent);
if( flag >= static_cast<int>(nNodes) ) {
nodes[flag-1].AddBckgrdWeight( weights.Get(iEvent), weights.GetOriginal(iEvent) );
nodes[flag-1].AddBckgrdWeight( weights.GetBoostWeight(iEvent), weights.GetOriginalWeight(iEvent) );
}
}

Expand Down Expand Up @@ -371,9 +371,9 @@ namespace FastBDT {
const unsigned int nEvents = sample.GetNEvents();
const unsigned int nSignals = sample.GetNSignals();
for(unsigned int iEvent = 0; iEvent < nSignals; ++iEvent)
weights.SetOriginal(iEvent, 2.0 * sums[1] / (sums[0] + sums[1]) * weights.GetOriginal(iEvent));
weights.SetOriginalWeight(iEvent, 2.0 * sums[1] / (sums[0] + sums[1]) * weights.GetOriginalWeight(iEvent));
for(unsigned int iEvent = nSignals; iEvent < nEvents; ++iEvent)
weights.SetOriginal(iEvent, 2.0 * sums[0] / (sums[0] + sums[1]) * weights.GetOriginal(iEvent));
weights.SetOriginalWeight(iEvent, 2.0 * sums[0] / (sums[0] + sums[1]) * weights.GetOriginalWeight(iEvent));
}

// Resize the FCache to the number of events, and initalise it with the inital 0.0 value
Expand Down Expand Up @@ -408,9 +408,9 @@ namespace FastBDT {
for(unsigned int iSpectator = 0; iSpectator < nSpectators; ++iSpectator) {
const uint64_t uniformBin = values.GetSpectator(iEvent, iSpectator);
if (iEvent < nSignals)
uniform_bin_weight_signal[iSpectator][uniformBin] += weights.GetOriginal(iEvent);
uniform_bin_weight_signal[iSpectator][uniformBin] += weights.GetOriginalWeight(iEvent);
else
uniform_bin_weight_bckgrd[iSpectator][uniformBin] += weights.GetOriginal(iEvent);
uniform_bin_weight_bckgrd[iSpectator][uniformBin] += weights.GetOriginalWeight(iEvent);
}
}
for(unsigned int iSpectator = 0; iSpectator < nSpectators; ++iSpectator) {
Expand Down Expand Up @@ -497,9 +497,9 @@ namespace FastBDT {
}

for(unsigned int iEvent = 0; iEvent < nSignals; ++iEvent)
weights.Set(iEvent, 2.0/(1.0+std::exp(2.0*FCache[iEvent])));
weights.SetBoostWeight(iEvent, 2.0/(1.0+std::exp(2.0*FCache[iEvent])));
for(unsigned int iEvent = nSignals; iEvent < nEvents; ++iEvent)
weights.Set(iEvent, 2.0/(1.0+std::exp(-2.0*FCache[iEvent])));
weights.SetBoostWeight(iEvent, 2.0/(1.0+std::exp(-2.0*FCache[iEvent])));

}

Expand Down Expand Up @@ -537,26 +537,20 @@ namespace FastBDT {
for(unsigned int iIndex = 0; iIndex < signal_event_index_sorted_by_F.size(); ++iIndex) {
unsigned int iEvent = signal_event_index_sorted_by_F[iIndex].index;

global_weight_below_current_F += weights.GetOriginal(iEvent);
global_weight_below_current_F += weights.GetOriginalWeight(iEvent);
double F = global_weight_below_current_F / sums[0];
double bw = weights.GetWithoutOriginal(iEvent);
double fw = 0.0;

for(unsigned int iSpectator = 0; iSpectator < nSpectators; ++iSpectator) {
const uint64_t uniformBin = values.GetSpectator(iEvent, iSpectator);
weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] += weights.GetOriginal(iEvent);
weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] += weights.GetOriginalWeight(iEvent);
double F_bin = weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] / (uniform_bin_weight_signal[iSpectator][uniformBin] * sums[0]);

fw += (F_bin - F);
}
fw *= flatnessLoss;
weights.SetFlatnessWeight(iEvent, fw);

if(bw + fw <= 0.0)
weights.Set(iEvent, 0.0);
else if (bw + fw >= 2.0)
weights.Set(iEvent, 2.0);
else
weights.Set(iEvent, bw + fw);
}

for(unsigned int iSpectator = 0; iSpectator < nSpectators; ++iSpectator) {
Expand All @@ -570,26 +564,19 @@ namespace FastBDT {
for(unsigned int iIndex = 0; iIndex < bckgrd_event_index_sorted_by_F.size(); ++iIndex) {
unsigned int iEvent = bckgrd_event_index_sorted_by_F[iIndex].index;

global_weight_below_current_F += weights.GetOriginal(iEvent);
global_weight_below_current_F += weights.GetOriginalWeight(iEvent);
double F = global_weight_below_current_F / sums[1];
double bw = weights.GetWithoutOriginal(iEvent);
double fw = 0.0;

for(unsigned int iSpectator = 0; iSpectator < nSpectators; ++iSpectator) {
const uint64_t uniformBin = values.GetSpectator(iEvent, iSpectator);
weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] += weights.GetOriginal(iEvent);
weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] += weights.GetOriginalWeight(iEvent);
double F_bin = weight_below_current_F_per_uniform_bin[iSpectator][uniformBin] / (uniform_bin_weight_bckgrd[iSpectator][uniformBin] * sums[1]);

fw += (F_bin - F);
}
fw *= flatnessLoss;

if(bw + fw <= 0.0)
weights.Set(iEvent, 0.0);
else if (bw + fw >= 2.0)
weights.Set(iEvent, 2.0);
else
weights.Set(iEvent, bw + fw);
weights.SetFlatnessWeight(iEvent, fw);

}

Expand Down
48 changes: 24 additions & 24 deletions src/test_FastBDT.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,8 +442,8 @@ class EventWeightsTest : public ::testing::Test {
virtual void SetUp() {
eventWeights = new EventWeights(10);
for(unsigned int i = 0; i < 10; ++i) {
eventWeights->Set(i, static_cast<Weight>(i+1));
eventWeights->SetOriginal(i, 2);
eventWeights->SetBoostWeight(i, static_cast<Weight>(i+1));
eventWeights->SetOriginalWeight(i, 2);
}
}

Expand All @@ -470,10 +470,10 @@ TEST_F(EventWeightsTest, WeightSumsAreNotInfluencedByZeroWeights) {
EventWeights *newEventWeights = new EventWeights(20);
for(unsigned int i = 0; i < 10; ++i) {
// Get delivers the weight*original weight, therefore we need to divide by the original weight afterwards
newEventWeights->Set(i*2, eventWeights->Get(i) / eventWeights->GetOriginal(i));
newEventWeights->SetOriginal(i*2, eventWeights->GetOriginal(i));
newEventWeights->Set(i*2 + 1, 0.0);
newEventWeights->SetOriginal(i*2 + 1, 0.0);
newEventWeights->SetBoostWeight(i*2, eventWeights->GetBoostWeight(i));
newEventWeights->SetOriginalWeight(i*2, eventWeights->GetOriginalWeight(i));
newEventWeights->SetBoostWeight(i*2 + 1, 0.0);
newEventWeights->SetOriginalWeight(i*2 + 1, 0.0);
}
auto newSums = newEventWeights->GetSums(10);
delete newEventWeights;
Expand All @@ -495,7 +495,7 @@ TEST_F(EventWeightsTest, GetterIsCorrect) {
TEST_F(EventWeightsTest, WeightSumsAndGetterAreCorrectlyUpdated) {

for(unsigned int i = 0; i < 10; ++i) {
eventWeights->Set(i, static_cast<Weight>(i+3));
eventWeights->SetBoostWeight(i, static_cast<Weight>(i+3));
}

auto sums = eventWeights->GetSums(5);
Expand Down Expand Up @@ -740,7 +740,7 @@ TEST_F(CumulativeDistributionsTest, NaNShouldBeIgnored) {
v[1] = eventSample->GetValues().Get(i, 1);
v[2] = eventSample->GetValues().GetSpectator(i, 0);
v[3] = eventSample->GetValues().GetSpectator(i, 1);
newEventSample->AddEvent(v, eventSample->GetWeights().GetOriginal(i), eventSample->IsSignal(i));
newEventSample->AddEvent(v, eventSample->GetWeights().GetOriginalWeight(i), eventSample->IsSignal(i));
newEventSample->AddEvent(std::vector<unsigned int>({0, 0, 1, 2}), 1.0, i < 50);
}
CumulativeDistributions newCDFsForLayer0(0, *newEventSample);
Expand Down Expand Up @@ -776,7 +776,7 @@ TEST_F(CumulativeDistributionsTest, ZeroWeightShouldBeIgnored) {
v[1] = eventSample->GetValues().Get(i, 1);
v[2] = eventSample->GetValues().GetSpectator(i, 0);
v[3] = eventSample->GetValues().GetSpectator(i, 1);
newEventSample->AddEvent(v, eventSample->GetWeights().GetOriginal(i), eventSample->IsSignal(i));
newEventSample->AddEvent(v, eventSample->GetWeights().GetOriginalWeight(i), eventSample->IsSignal(i));
newEventSample->AddEvent(std::vector<unsigned int>({i%2 + 1, i%3 + 1, 1, 2}), 0.0, i < 50);
}
CumulativeDistributions newCDFsForLayer0(0, *newEventSample);
Expand Down Expand Up @@ -1046,16 +1046,16 @@ TEST_F(NodeTest, NegativeWeightsAreHandledCorrectly) {

Node node(0,0);
node.SetWeights({0.0, 0.0, 0.0});
node.AddSignalWeight(-2.0, -1.0);
node.AddSignalWeight(-4.0, -1.0);
node.AddBckgrdWeight(-4.0, -1.0);
node.AddSignalWeight(2.0, -1.0);
node.AddSignalWeight(4.0, -1.0);
node.AddBckgrdWeight(4.0, -1.0);
EXPECT_FLOAT_EQ(node.GetPurity(), 0.6);
EXPECT_FLOAT_EQ(node.GetBoostWeight(), -0.125);

node.SetWeights({0.0, 0.0, 0.0});
node.AddSignalWeight(-2.0, 1.0);
node.AddSignalWeight(1.0, -2.0);
node.AddBckgrdWeight(0.5, -0.5);
node.AddSignalWeight(-0.5, -2.0);
node.AddBckgrdWeight(-1.0, -0.5);
// Purity above 1.0 can happen with negative weights
EXPECT_FLOAT_EQ(node.GetPurity(), 2.0);
EXPECT_FLOAT_EQ(node.GetBoostWeight(), 0.375);
Expand Down Expand Up @@ -1187,14 +1187,14 @@ class TreeBuilderTest : public ::testing::Test {
eventSample->AddEvent( std::vector<unsigned int>({ 2, 2 }), 1.0, false);

auto &weights = eventSample->GetWeights();
weights.Set(0, 4.0);
weights.Set(1, 1.0);
weights.Set(2, 2.0);
weights.Set(3, 3.0);
weights.Set(4, 2.0);
weights.Set(5, 1.0);
weights.Set(6, 3.0);
weights.Set(7, 4.0);
weights.SetBoostWeight(0, 4.0);
weights.SetBoostWeight(1, 1.0);
weights.SetBoostWeight(2, 2.0);
weights.SetBoostWeight(3, 3.0);
weights.SetBoostWeight(4, 2.0);
weights.SetBoostWeight(5, 1.0);
weights.SetBoostWeight(6, 3.0);
weights.SetBoostWeight(7, 4.0);
}

virtual void TearDown() {
Expand Down Expand Up @@ -1657,8 +1657,8 @@ TEST_F(CornerCasesTest, PerfectSeparationWithDifferentWeights) {
const unsigned int nEvents = sample->GetNEvents();
for(unsigned int iEvent = 0; iEvent < nEvents; ++iEvent) {
if(iEvent < nSignals)
sig += sample->GetWeights().GetOriginal(iEvent);
tot += sample->GetWeights().GetOriginal(iEvent);
sig += sample->GetWeights().GetOriginalWeight(iEvent);
tot += sample->GetWeights().GetOriginalWeight(iEvent);
}

// Train without randomness and only with one layer per tree and shrinkage 1
Expand Down

0 comments on commit 9a38430

Please sign in to comment.