diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md
index a60fbbba..4e9eb059 100644
--- a/doc/releases/changelog-dev.md
+++ b/doc/releases/changelog-dev.md
@@ -6,6 +6,14 @@
## Bug fixes
+ * Fixed destructor in `RegionSelectionManager` so that `RegionSelection`
+ objects allocated inside the `region_vector` are properly destructed upon
+ existing `scope/destruction` of `RegionSelectionManager`.
+ ([#113](https://github.com/MadAnalysis/madanalysis5/pull/113))
+
+
## Contributors
-This release contains contributions from (in alphabetical order):
\ No newline at end of file
+This release contains contributions from (in alphabetical order):
+
+[Kyle Fan](https://github.com/kfan326)
diff --git a/madanalysis/IOinterface/job_reader.py b/madanalysis/IOinterface/job_reader.py
index 30bb01e3..112e9f67 100644
--- a/madanalysis/IOinterface/job_reader.py
+++ b/madanalysis/IOinterface/job_reader.py
@@ -30,6 +30,12 @@
from madanalysis.layout.histogram import Histogram
from madanalysis.layout.histogram_logx import HistogramLogX
from madanalysis.layout.histogram_frequency import HistogramFrequency
+from madanalysis.IOinterface.sqlite_reader import getMeanAndStdev
+from madanalysis.IOinterface.sqlite_reader import DBreader_debug
+from madanalysis.IOinterface.sqlite_reader import getHistoStatisticsAvg
+
+
+
import glob
import logging
import shutil
@@ -318,10 +324,18 @@ def ExtractHistos(self,dataset,plot,merging=False):
while(os.path.isdir(self.safdir+"/"+name+"/MergingPlots_"+str(i))):
i+=1
filename = self.safdir+"/"+name+"/MergingPlots_"+str(i-1)+"/Histograms/histos.saf"
+ ##file path for sqlite db
+ sqlite_db_filename = self.safdir+"/"+name+"/MergingPlots_"+str(i-1)+"/Histograms/histo.db"
+
+
+
else:
while(os.path.isdir(self.safdir+"/"+name+"/MadAnalysis5job_"+str(i))):
i+=1
filename = self.safdir+"/"+name+"/MadAnalysis5job_"+str(i-1)+"/Histograms/histos.saf"
+ ##file path for sqlite db
+ sqlite_db_filename = self.safdir+"/"+name+"/MadAnalysis5job_"+str(i-1)+"/Histograms/histo.db"
+
# Opening the file
try:
@@ -348,6 +362,16 @@ def ExtractHistos(self,dataset,plot,merging=False):
data_negative = []
labels = []
+
+ ## SqliteDB extractor
+ sqlite_exists = os.path.isfile(sqlite_db_filename)
+ if sqlite_exists:
+ sqlite_output_dictionary = getMeanAndStdev(sqlite_db_filename)
+ histoStatistics = getHistoStatisticsAvg(sqlite_db_filename)
+ #DBreader_debug(sqlite_output_dictionary)
+
+
+
# Loop over the lines
numline=0
for line in file:
@@ -391,13 +415,40 @@ def ExtractHistos(self,dataset,plot,merging=False):
elif words[0].lower()=='':
histoTag.activate()
elif words[0].lower()=='':
+
+
histoTag.desactivate()
plot.histos.append(copy.copy(histoinfo))
- plot.histos[-1].positive.array = data_positive[:]
- plot.histos[-1].negative.array = data_negative[:]
- histoinfo.Reset()
- data_positive = []
- data_negative = []
+
+ if not sqlite_exists:
+
+ plot.histos[-1].positive.array = data_positive[:]
+ plot.histos[-1].negative.array = data_negative[:]
+ histoinfo.Reset()
+ bin_means = []
+ bin_stdev = []
+ else:
+ bin_means = []
+ bin_stdev = []
+
+ # save bin mean and stdev into histogram_core for positive and negative values
+ for bin_index in sqlite_output_dictionary[histoinfo.name]:
+ if bin_index not in ["underflow", "overflow"]:
+ bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0])
+ bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1])
+ elif bin_index == "underflow":
+ plot.histos[-1].positive.underflow = sqlite_output_dictionary[histoinfo.name][bin_index][0]
+ elif bin_index == "overflow":
+ plot.histos[-1].positive.overflow = sqlite_output_dictionary[histoinfo.name][bin_index][0]
+
+ histoinfo.Reset()
+
+ plot.histos[-1].positive.array = bin_means[:]
+ ##plot.histos[-1].negative.array = bin_means[:]
+ plot.histos[-1].positive.stdev = bin_stdev[:]
+ ##plot.histos[-1].negative.stdev = bin_stdev[:]
+
+
elif words[0].lower()=='':
histoFreqTag.activate()
elif words[0].lower()=='':
@@ -415,11 +466,35 @@ def ExtractHistos(self,dataset,plot,merging=False):
elif words[0].lower()=='':
histoLogXTag.desactivate()
plot.histos.append(copy.copy(histologxinfo))
- plot.histos[-1].positive.array = data_positive[:]
- plot.histos[-1].negative.array = data_negative[:]
- histologxinfo.Reset()
- data_positive = []
- data_negative = []
+
+ if not sqlite_exists:
+
+ plot.histos[-1].positive.array = data_positive[:]
+ plot.histos[-1].negative.array = data_negative[:]
+ histologxinfo.Reset()
+ data_positive = []
+ data_negative = []
+
+ else:
+ bin_means = []
+ bin_stdev = []
+
+ # save bin mean and stdev into histogram_core for positive and negative values
+ for bin_index in sqlite_output_dictionary[histologxinfo.name]:
+ if bin_index not in ["underflow", "overflow"]:
+ bin_means.append(sqlite_output_dictionary[histoinfo.name][bin_index][0])
+ bin_stdev.append(sqlite_output_dictionary[histoinfo.name][bin_index][1])
+ elif bin_index == "underflow":
+ plot.histos[-1].positive.underflow = sqlite_output_dictionary[histologxinfo.name][bin_index][0]
+ elif bin_index == "overflow":
+ plot.histos[-1].positive.overflow = sqlite_output_dictionary[histologxinfo.name][bin_index][0]
+
+ histologxinfo.Reset()
+
+ plot.histos[-1].positive.array = bin_means[:]
+ ##plot.histos[-1].negative.array = bin_means[:]
+ plot.histos[-1].positive.stdev = bin_stdev[:]
+ ##plot.histos[-1].negative.stdev = bin_stdev[:]
# Looking from histogram description
elif descriptionTag.activated:
@@ -464,80 +539,143 @@ def ExtractHistos(self,dataset,plot,merging=False):
# Looking from histogram statistics
elif statisticsTag.activated and len(words)==2:
+
if statisticsTag.Nlines==0:
results = self.ExtractStatisticsInt(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.nevents=results[0]
- histoinfo.negative.nevents=results[1]
+ if sqlite_exists:
+ histoinfo.positive.nevents = int(histoStatistics[histoinfo.name][0])
+ histoinfo.negative.nevents = int(histoStatistics[histoinfo.name][1])
+ else :
+ histoinfo.positive.nevents=results[0]
+ histoinfo.negative.nevents=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.nevents=results[0]
- histologxinfo.negative.nevents=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.nevents = int(histoStatistics[histologxinfo.name][0])
+ histologxinfo.negative.nevents = int(histoStatistics[histologxinfo.name][1])
+ else :
+ histologxinfo.positive.nevents=results[0]
+ histologxinfo.negative.nevents=results[1]
elif histoFreqTag.activated:
histofreqinfo.positive.nevents=results[0]
histofreqinfo.negative.nevents=results[1]
elif statisticsTag.Nlines==1:
results = self.ExtractStatisticsFloat(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.sumwentries=results[0]
- histoinfo.negative.sumwentries=results[1]
+ if sqlite_exists:
+ histoinfo.positive.sumwentries = histoStatistics[histoinfo.name][2]
+ histoinfo.negative.sumwentries = histoStatistics[histoinfo.name][3]
+ else :
+ histoinfo.positive.sumwentries=results[0]
+ histoinfo.negative.sumwentries=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.sumwentries=results[0]
- histologxinfo.negative.sumwentries=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.sumwentries = histoStatistics[histologxinfo.name][2]
+ histologxinfo.negative.sumwentries = histoStatistics[histologxinfo.name][3]
+ else :
+ histologxinfo.positive.sumwentries=results[0]
+ histologxinfo.negative.sumwentries=results[1]
elif histoFreqTag.activated:
histofreqinfo.positive.sumwentries=results[0]
histofreqinfo.negative.sumwentries=results[1]
elif statisticsTag.Nlines==2:
results = self.ExtractStatisticsInt(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.nentries=results[0]
- histoinfo.negative.nentries=results[1]
+ if sqlite_exists:
+ histoinfo.positive.nentries = int(histoStatistics[histoinfo.name][4])
+ histoinfo.negative.nentries = int(histoStatistics[histoinfo.name][5])
+ else :
+ histoinfo.positive.nentries=results[0]
+ histoinfo.negative.nentries=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.nentries=results[0]
- histologxinfo.negative.nentries=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.nentries = int(histoStatistics[histologxinfo.name][4])
+ histologxinfo.negative.nentries = int(histoStatistics[histologxinfo.name][5])
+ else :
+ histologxinfo.positive.nentries=results[0]
+ histologxinfo.negative.nentries=results[1]
elif histoFreqTag.activated:
histofreqinfo.positive.nentries=results[0]
histofreqinfo.negative.nentries=results[1]
elif statisticsTag.Nlines==3:
results = self.ExtractStatisticsFloat(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.sumw=results[0]
- histoinfo.negative.sumw=results[1]
+ if sqlite_exists:
+ histoinfo.positive.sumw = histoStatistics[histoinfo.name][6]
+ histoinfo.negative.sumw = histoStatistics[histoinfo.name][7]
+ else :
+ histoinfo.positive.sumw=results[0]
+ histoinfo.negative.sumw=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.sumw=results[0]
- histologxinfo.negative.sumw=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.sumw = histoStatistics[histologxinfo.name][6]
+ histologxinfo.negative.sumw = histoStatistics[histologxinfo.name][7]
+ else :
+ histologxinfo.positive.sumw=results[0]
+ histologxinfo.negative.sumw=results[1]
elif histoFreqTag.activated:
histofreqinfo.positive.sumw=results[0]
histofreqinfo.negative.sumw=results[1]
elif statisticsTag.Nlines==4 and not histoFreqTag.activated:
results = self.ExtractStatisticsFloat(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.sumw2=results[0]
- histoinfo.negative.sumw2=results[1]
+ if sqlite_exists:
+ histoinfo.positive.sumw2 = histoStatistics[histoinfo.name][8]
+ histoinfo.negative.sumw2 = histoStatistics[histoinfo.name][9]
+ else :
+ histoinfo.positive.sumw2=results[0]
+ histoinfo.negative.sumw2=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.sumw2=results[0]
- histologxinfo.negative.sumw2=results[1]
-
+ if sqlite_exists:
+ histologxinfo.positive.sumw2 = histoStatistics[histologxinfo.name][8]
+ histologxinfo.negative.sumw2 = histoStatistics[histologxinfo.name][9]
+ else :
+ histologxinfo.positive.sumw2=results[0]
+ histologxinfo.negative.sumw2=results[1]
elif statisticsTag.Nlines==5 and not histoFreqTag.activated:
results = self.ExtractStatisticsFloat(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.sumwx=results[0]
- histoinfo.negative.sumwx=results[1]
+ if sqlite_exists:
+ histoinfo.positive.sumwx = histoStatistics[histoinfo.name][10]
+ histoinfo.negative.sumwx = histoStatistics[histoinfo.name][11]
+ else :
+ histoinfo.positive.sumwx=results[0]
+ histoinfo.negative.sumwx=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.sumwx=results[0]
- histologxinfo.negative.sumwx=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.sumwx = histoStatistics[histologxinfo.name][10]
+ histologxinfo.negative.sumwx = histoStatistics[histologxinfo.name][11]
+ else :
+ histologxinfo.positive.sumwx=results[0]
+ histologxinfo.negative.sumwx=results[1]
elif statisticsTag.Nlines==6 and not histoFreqTag.activated:
results = self.ExtractStatisticsFloat(words,numline,filename)
+
if histoTag.activated:
- histoinfo.positive.sumw2x=results[0]
- histoinfo.negative.sumw2x=results[1]
+ if sqlite_exists:
+ histoinfo.positive.sumw2x = histoStatistics[histoinfo.name][12]
+ histoinfo.negative.sumw2x = histoStatistics[histoinfo.name][13]
+ else :
+ histoinfo.positive.sumw2x=results[0]
+ histoinfo.negative.sumw2x=results[1]
elif histoLogXTag.activated:
- histologxinfo.positive.sumw2x=results[0]
- histologxinfo.negative.sumw2x=results[1]
+ if sqlite_exists:
+ histologxinfo.positive.sumw2x = histoStatistics[histologxinfo.name][12]
+ histologxinfo.negative.sumw2x = histoStatistics[histologxinfo.name][13]
+ else :
+ histologxinfo.positive.sumw2x=results[0]
+ histologxinfo.negative.sumw2x=results[1]
else:
logging.getLogger('MA5').warning('Extra line is found: '+line)
diff --git a/madanalysis/IOinterface/job_writer.py b/madanalysis/IOinterface/job_writer.py
index 07941a7d..cb72197c 100644
--- a/madanalysis/IOinterface/job_writer.py
+++ b/madanalysis/IOinterface/job_writer.py
@@ -750,6 +750,7 @@ def WriteMakefiles(self, option="", **kwargs):
options.has_root_inc = self.main.archi_info.has_root
options.has_root_lib = self.main.archi_info.has_root
+ options.has_sqlite = self.main.archi_info.has_sqlite3
#options.has_userpackage = True
toRemove=['Log/compilation.log','Log/linking.log','Log/cleanup.log','Log/mrproper.log']
diff --git a/madanalysis/IOinterface/library_writer.py b/madanalysis/IOinterface/library_writer.py
index 39812f29..bdc74271 100644
--- a/madanalysis/IOinterface/library_writer.py
+++ b/madanalysis/IOinterface/library_writer.py
@@ -131,7 +131,7 @@ def WriteMakefileForInterfaces(self,package):
filename = self.path+"/SampleAnalyzer/Test/Makefile_delphesMA5tune"
elif package=='test_root':
filename = self.path+"/SampleAnalyzer/Test/Makefile_root"
-
+
# Header
title=''
if package=='commons':
@@ -239,10 +239,13 @@ def WriteMakefileForInterfaces(self,package):
options.ma5_fastjet_mode = self.main.archi_info.has_fastjet
options.has_fastjet_inc = self.main.archi_info.has_fastjet
options.has_fastjet_lib = self.main.archi_info.has_fastjet
+ #options.has_sqlite_lib = self.main.archi_info.has_sqlite3
+ options.has_sqlite_tag = self.main.archi_info.has_sqlite3
# options.has_fastjet_ma5lib = self.main.archi_info.has_fastjet
toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log'])
elif package=='test_commons':
options.has_commons = True
+ options.has_sqlite_tag = self.main.archi_info.has_sqlite3
toRemove.extend(['compilation_commons.log','linking_commons.log','cleanup_commons.log','mrproper_commons.log','../Bin/TestCommons.log'])
elif package=='zlib':
options.has_commons = True
@@ -252,6 +255,7 @@ def WriteMakefileForInterfaces(self,package):
elif package=='test_zlib':
options.has_commons = True
options.has_zlib_ma5lib = True
+ options.has_sqlite_tag = self.main.archi_info.has_sqlite3
# options.has_zlib_lib = True
toRemove.extend(['compilation_zlib.log','linking_zlib.log','cleanup_zlib.log','mrproper_zlib.log','../Bin/TestZlib.log'])
elif package=='delphes':
@@ -324,6 +328,8 @@ def WriteMakefileForInterfaces(self,package):
options.has_fastjet_lib = self.main.archi_info.has_fastjet
options.ma5_fastjet_mode = self.main.archi_info.has_fastjet
options.has_substructure = self.main.archi_info.has_fjcontrib and self.main.archi_info.has_fastjet
+ options.has_sqlite_tag = self.main.archi_info.has_sqlite3
+ options.has_sqlite_lib = self.main.archi_info.has_sqlite3
toRemove.extend(['compilation.log','linking.log','cleanup.log','mrproper.log'])
elif package=='test_process':
@@ -342,6 +348,8 @@ def WriteMakefileForInterfaces(self,package):
# options.has_delphesMA5tune_tag = self.main.archi_info.has_delphesMA5tune
# options.has_zlib_tag = self.main.archi_info.has_zlib
toRemove.extend(['compilation_process.log','linking_process.log','cleanup_process.log','mrproper_process.log','../Bin/TestSampleAnalyzer.log'])
+ elif package=='sqlite':
+ options.has_sqlite = self.main.archi_info.has_sqlite3
# file pattern
if package in ['commons','process','configuration']:
@@ -373,7 +381,7 @@ def WriteMakefileForInterfaces(self,package):
hfiles = ['DelphesMA5tune/*.h']
elif package=='test_root':
cppfiles = ['Root/*.cpp']
- hfiles = ['Root/*.h']
+ hfiles = ['Root/*.h']
else:
cppfiles = [package+'/*.cpp']
hfiles = [package+'/*.h']
diff --git a/madanalysis/IOinterface/sqlite_reader.py b/madanalysis/IOinterface/sqlite_reader.py
new file mode 100644
index 00000000..60ff5bb4
--- /dev/null
+++ b/madanalysis/IOinterface/sqlite_reader.py
@@ -0,0 +1,193 @@
+import sqlite3
+from matplotlib import pyplot as plt
+import numpy as np
+import math
+import statistics
+
+
+def getMeanAndStdevOld(path):
+
+ con = sqlite3.connect(path)
+ cursor = con.cursor()
+
+ bin_data = cursor.execute("select * from data;").fetchall()
+
+ pos_bins = dict()
+ neg_bins = dict()
+
+ ## bin_data has all data for the histogram, need to get mean and standard deviation for each bin
+ ## each row of the query is a tuple of 5 elements [histo name, weight id, bin #, positive value, negative value]
+ ## sort them into +bin/-bin[name] -> bin # -> [mean, standard deviation]
+
+ for row in bin_data:
+ ## if the histo name is not inside the bin dictionaries, create a new dictionary for each of +/- bin dictionary
+ ## append values to +/-bin[name][bin#]
+
+ if row[0] not in pos_bins or row[0] not in neg_bins:
+ pos_bins[row[0]] = dict()
+ neg_bins[row[0]] = dict()
+ pos_bins[row[0]][row[2]] = [float(row[3])]
+ neg_bins[row[0]][row[2]] = [float(row[4])]
+
+ else:
+ if row[2] in pos_bins[row[0]] or row[2] in neg_bins[row[0]]:
+ pos_bins[row[0]][row[2]].append(float(row[3]))
+ neg_bins[row[0]][row[2]].append(float(row[4]))
+ else :
+ pos_bins[row[0]][row[2]] = [float(row[3])]
+ neg_bins[row[0]][row[2]] = [float(row[4])]
+
+ output = dict()
+
+ for histo_name in pos_bins:
+ output[histo_name] = dict()
+ for bin_i in pos_bins[histo_name]:
+ output[histo_name][bin_i] = [statistics.mean(pos_bins[histo_name][bin_i]), statistics.stdev(pos_bins[histo_name][bin_i])]
+
+ for histo_name in neg_bins:
+ for bin_i in neg_bins[histo_name]:
+ output[histo_name][bin_i].extend([statistics.mean(neg_bins[histo_name][bin_i]), statistics.stdev(neg_bins[histo_name][bin_i])])
+
+ return output
+
+
+def getStatistics(stats):
+ histoname_dict = dict()
+ for entry in stats:
+ if entry[0] not in histoname_dict:
+ histoname_dict[entry[0]] = dict()
+ sumw = float(entry[2]) - abs(float(entry[3]))
+ if sumw == 0:
+ raise Exception("sumw is 0 for histo " + entry[0] + " and weight id " + entry[1])
+ histoname_dict[entry[0]][entry[1]] = sumw
+ return histoname_dict
+
+
+def getMeanAndStdev(path):
+
+ con = sqlite3.connect(path)
+ cursor = con.cursor()
+ bin_data = cursor.execute("select * from data;").fetchall()
+ stats_data = cursor.execute("select name, id, pos_sum_event_weights_over_events, neg_sum_event_weights_over_events from Statistics").fetchall()
+
+ statsdict = getStatistics(stats_data)
+
+
+ ## parse data in the form of parsed_data[histo_name][bin #][{positive value, negative value}]
+ parsed_data = dict()
+ for row in bin_data:
+
+ histo_name = row[0]
+ weight_id = row[1]
+ bin_number = row[2]
+ sumw = statsdict[histo_name][str(weight_id)]
+
+ value = (float(row[3]) - abs(float(row[4]))) /sumw
+ if histo_name not in parsed_data:
+ ## if histo name is not in the parsed_data dictionary, then create a new bin dictionary for that histo, then for the bin, create a weigh id dictionary
+ parsed_data[histo_name] = dict()
+ parsed_data[histo_name][bin_number] = []
+
+ else:
+ ## since histo name is in the parsed_data dictionary, we need to check if the bin in the dictioary, if not then create a weight id dictionary for that bin
+ if bin_number not in parsed_data[histo_name]:
+ parsed_data[histo_name][bin_number] = []
+
+ parsed_data[histo_name][bin_number].append(value)
+
+ output = dict()
+ for histo_name in parsed_data:
+ output[histo_name] = dict()
+ for bin_number in parsed_data[histo_name]:
+ output[histo_name][bin_number] = [statistics.mean(parsed_data[histo_name][bin_number]), statistics.stdev(parsed_data[histo_name][bin_number])]
+
+ return output
+
+def getHistoStatisticsAvg(path):
+
+ con = sqlite3.connect(path)
+ cursor = con.cursor()
+
+
+ statistics = cursor.execute("select name, avg(pos_num_events), avg(neg_num_events), avg(pos_sum_event_weights_over_events), avg(neg_sum_event_weights_over_events), avg(pos_entries), avg(neg_entries), avg(pos_sum_event_weights_over_entries), avg(neg_sum_event_weights_over_entries), avg(pos_sum_squared_weights), avg(neg_sum_squared_weights), avg(pos_value_times_weight), avg(neg_value_times_weight), avg(pos_value_squared_times_weight), avg(neg_value_squared_times_weight) from Statistics group by name;").fetchall()
+
+ statdict = dict()
+ for i in range(len(statistics)):
+ statdict[statistics[i][0]] = statistics[i][1:]
+
+ return statdict;
+
+
+
+
+
+
+## debug for printing out output dictionary
+## structure is as follows:
+## output[histogram_name][bin #] = [positive mean, positive stdev, negative mean, negative stddev]
+
+
+def DBreader_debug(output):
+
+ for name in output:
+ print(name)
+ for eachbin in output[name]:
+ print(eachbin)
+ for val in output[name][eachbin]:
+ print(val)
+
+
+ for histo in output:
+ num_of_keys = len(output[histo].keys())
+ labels = [None] * num_of_keys
+ for i in range(1,num_of_keys):
+ labels[i] = i
+ labels[0] = 'underflow'
+ labels[num_of_keys-1] = 'overflow'
+ positives = [None] * num_of_keys
+ negatives = [None] * num_of_keys
+ for row in output[histo]:
+ if(row == 'underflow'):
+ positives[0] = output[histo][row][0]
+ negatives[0] = output[histo][row][2]
+ elif(row == 'overflow'):
+ positives[num_of_keys-1] = output[histo][row][0]
+ negatives[num_of_keys-1] = output[histo][row][2]
+ else:
+ positives[int(row)] = output[histo][row][0]
+ negatives[int(row)] = output[histo][row][2]
+ #for lable in lables:
+ # print(lable)
+ #for val in positives:
+ # print(val)
+ #for val in negatives:
+ # print(val)
+ x = np.arange(num_of_keys)
+ width = 0.5
+ fig, ax = plt.subplots()
+ rects1 = ax.bar(x - width/3, positives, width, label="positives avg")
+ rects2 = ax.bar(x + width/3, negatives, width, label="negatives avg")
+
+ ax.set_ylabel('Events Luminosity = ')
+ ax.set_title(histo)
+ ax.set_xticks(x, labels, rotation = 65)
+ ax.legend()
+
+ #ax.bar_label(rects1, padding=3)
+ #ax.bar_label(rects2, padding=3)
+
+ fig.tight_layout()
+ plt.show()
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/madanalysis/build/makefile_writer.py b/madanalysis/build/makefile_writer.py
index f3ff61f4..208b3954 100644
--- a/madanalysis/build/makefile_writer.py
+++ b/madanalysis/build/makefile_writer.py
@@ -41,6 +41,7 @@ def __init__(self):
self.has_fastjet = False
self.has_delphes = False
self.has_delphesMA5tune = False
+ self.has_sqlite3 = False
@staticmethod
@@ -98,7 +99,10 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
file.write('\tcd Test && $(MAKE) -f Makefile_delphesMA5tune\n')
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile\n')
- file.write('\tcd Test && $(MAKE) -f Makefile_process\n')
+ file.write('\tcd Test && $(MAKE) -f Makefile_process\n')
+ if options.has_sqlite3:
+ file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite\n')
+ file.write('\tcd Test && $(MAKE) -f Makefile_sqlite\n')
file.write('\n')
# Clean
@@ -125,6 +129,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile clean\n')
file.write('\tcd Test && $(MAKE) -f Makefile_process clean\n')
+ if options.has_sqlite3:
+ file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite clean\n')
+ file.write('\tcd Test && $(MAKE) -f Makefile_sqlite clean\n')
file.write('\n')
# Mrproper
@@ -152,6 +159,9 @@ def UserfriendlyMakefileForSampleAnalyzer(filename,options):
if options.has_process:
file.write('\tcd Process && $(MAKE) -f Makefile mrproper\n')
file.write('\tcd Test && $(MAKE) -f Makefile_process mrproper\n')
+ if options.has_sqlite3:
+ file.write('\tcd Interfaces && $(MAKE) -f Makefile_sqlite mrproper\n')
+ file.write('\tcd Test && $(MAKE) -f Makefile_sqlite mrproper\n')
file.write('\n')
# Closing the file
@@ -194,6 +204,9 @@ def __init__(self):
self.has_root_tag = False
self.has_root_lib = False
self.has_root_ma5lib = False
+ self.has_sqlite = False
+ self.has_sqlite_tag = False
+ self.has_sqlite_lib = False
@@ -321,7 +334,9 @@ def Makefile(
for header in archi_info.delphesMA5tune_inc_paths:
cxxflags.extend(['-I'+header])
file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n')
-
+
+
+
# - tags
cxxflags=[]
if options.has_root_tag:
@@ -338,6 +353,8 @@ def Makefile(
cxxflags.extend(['-DDELPHES_USE'])
if options.has_delphesMA5tune_tag:
cxxflags.extend(['-DDELPHESMA5TUNE_USE'])
+ if options.has_sqlite_tag:
+ cxxflags.extend(['-DSQLITE3_USE'])
if len(cxxflags)!=0:
file.write('CXXFLAGS += '+' '.join(cxxflags)+'\n')
file.write('\n')
@@ -347,7 +364,9 @@ def Makefile(
# - general
libs=[]
- file.write('LIBFLAGS = \n')
+
+ # added SQL
+ #file.write('LIBFLAGS = -l sqlite3\n')
# - commons
if options.has_commons:
@@ -429,6 +448,14 @@ def Makefile(
if options.has_heptoptagger:
file.write('LIBFLAGS += -lHEPTopTagger_for_ma5\n')
+ # SQLite3
+ if options.has_sqlite:
+ file.write('LIBFLAGS += -l sqlite3\n')
+
+ if options.has_sqlite_lib:
+ file.write('LIBFLAGS += -l sqlite_for_ma5\n')
+
+
# - Commons
if options.has_commons:
libs=[]
@@ -464,6 +491,8 @@ def Makefile(
libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsubstructure_for_ma5.so')
if options.has_heptoptagger:
libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libHEPTopTagger_for_ma5.so')
+ if options.has_sqlite_lib:
+ libs.append('$(MA5_BASE)/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so')
if len(libs)!=0:
file.write('# Requirements to check before building\n')
for ind in range(0,len(libs)):
diff --git a/madanalysis/core/library_builder.py b/madanalysis/core/library_builder.py
index cdb2c685..d6f476f0 100644
--- a/madanalysis/core/library_builder.py
+++ b/madanalysis/core/library_builder.py
@@ -80,6 +80,10 @@ def checkMA5(self):
libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphes_for_ma5.so')
if self.archi_info.has_delphesMA5tune:
libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.so')
+ if self.archi_info.has_sqlite3:
+ libraries.append(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so')
+
+
for library in libraries:
if not os.path.isfile(library):
self.logger.debug('\t-> library '+ library + " not found.")
diff --git a/madanalysis/core/main.py b/madanalysis/core/main.py
index 570b539a..85ec8876 100644
--- a/madanalysis/core/main.py
+++ b/madanalysis/core/main.py
@@ -626,6 +626,13 @@ def BuildLibrary(self,forced=False):
self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestRoot',
self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True])
+ # SQLite3
+ if self.archi_info.has_sqlite3:
+ libraries.append(['SQLite3', 'interface to SQLite3', 'sqlite',
+ self.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libsqlite_for_ma5.so',
+ self.archi_info.ma5dir + '/tools/SampleAnalyzer/Interfaces', False])
+
+
# Process
libraries.append(['process', 'SampleAnalyzer core', 'process',
self.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libprocess_for_ma5.so',
@@ -634,6 +641,7 @@ def BuildLibrary(self,forced=False):
self.archi_info.ma5dir + '/tools/SampleAnalyzer/Bin/TestSampleAnalyzer',
self.archi_info.ma5dir + '/tools/SampleAnalyzer/Test/', True])
+
# Writing the Makefiles
self.logger.info("")
@@ -659,7 +667,8 @@ def BuildLibrary(self,forced=False):
options.has_zlib = self.archi_info.has_zlib
options.has_fastjet = self.archi_info.has_fastjet
options.has_delphes = self.archi_info.has_delphes
- options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune
+ options.has_delphesMA5tune = self.archi_info.has_delphesMA5tune
+ options.has_sqlite3 = self.archi_info.has_sqlite3
#MakefileWriter.UserfriendlyMakefileForSampleAnalyzer(self.archi_info.ma5dir+'/tools/SampleAnalyzer/Makefile',options)
# Writing the setup
diff --git a/madanalysis/job/job_execute.py b/madanalysis/job/job_execute.py
index 6380d211..9137612b 100644
--- a/madanalysis/job/job_execute.py
+++ b/madanalysis/job/job_execute.py
@@ -43,7 +43,7 @@ def WriteExecute(file,main,part_list):
file.write(' if (weighted_events_ && event.mc()!=0) ' +\
'__event_weight__ = event.mc()->weight();\n\n')
file.write(' if (sample.mc()!=0) sample.mc()->addWeightedEvents(__event_weight__);\n')
- file.write(' Manager()->InitializeForNewEvent(__event_weight__);\n')
+ file.write(' Manager()->InitializeForNewEvent(__event_weight__, event.mc()->multiweights().GetWeights());\n')
file.write('\n')
# Reseting instance name
diff --git a/madanalysis/layout/histogram.py b/madanalysis/layout/histogram.py
index 949eaf8b..0c4992b5 100644
--- a/madanalysis/layout/histogram.py
+++ b/madanalysis/layout/histogram.py
@@ -85,7 +85,7 @@ def FinalizeReading(self,main,dataset):
# Data
data = []
for i in range(0,len(self.positive.array)):
- data.append(self.positive.array[i]-self.negative.array[i])
+ data.append(self.positive.array[i])
if data[-1]<0:
self.warnings.append(\
'dataset='+dataset.name+\
@@ -95,6 +95,14 @@ def FinalizeReading(self,main,dataset):
data[-1]=0
self.summary.array = data[:] # [:] -> clone of data
+ #stdev
+ stdev_data = []
+ for i in range(0, len(self.positive.array)):
+ stdev_data.append(max(0, self.positive.stdev[i]))
+ self.summary.stdev = stdev_data[:]
+
+
+
# Integral
self.positive.ComputeIntegral()
self.negative.ComputeIntegral()
diff --git a/madanalysis/layout/histogram_core.py b/madanalysis/layout/histogram_core.py
index 0a3355a4..bb566599 100644
--- a/madanalysis/layout/histogram_core.py
+++ b/madanalysis/layout/histogram_core.py
@@ -50,6 +50,7 @@ def __init__(self):
self.nan = 0.
self.inf = 0.
self.array = []
+ self.stdev = []
def ComputeIntegral(self):
diff --git a/madanalysis/layout/histogram_frequency_core.py b/madanalysis/layout/histogram_frequency_core.py
index d797d74a..377fa93e 100644
--- a/madanalysis/layout/histogram_frequency_core.py
+++ b/madanalysis/layout/histogram_frequency_core.py
@@ -35,6 +35,7 @@ def __init__(self):
self.overflow = 0.
self.underflow = 0.
self.array = []
+ self.stdev = []
def ComputeIntegral(self):
self.integral = 0
diff --git a/madanalysis/layout/histogram_logx.py b/madanalysis/layout/histogram_logx.py
index bb16edbf..386be6f5 100644
--- a/madanalysis/layout/histogram_logx.py
+++ b/madanalysis/layout/histogram_logx.py
@@ -111,6 +111,8 @@ def FinalizeReading(self,main,dataset):
data[-1]=0
self.summary.array = data[:] # [:] -> clone of data
+
+
# Integral
self.positive.ComputeIntegral()
self.negative.ComputeIntegral()
diff --git a/madanalysis/layout/plotflow.py b/madanalysis/layout/plotflow.py
index 8fe99d39..1535b443 100644
--- a/madanalysis/layout/plotflow.py
+++ b/madanalysis/layout/plotflow.py
@@ -640,8 +640,17 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames):
outputPy.write(str(histos[ind].summary.array[bin-1]*scales[ind]))
outputPy.write('])\n\n')
+ #stdev
+ for ind in range(0, len(histos)):
+ outputPy.write(' # Creating array for stdev\n')
+ outputPy.write(' err = numpy.array([')
+ for bin in range(1, xnbin+1):
+ if bin !=1:
+ outputPy.write(',')
+ outputPy.write(str(histos[ind].summary.stdev[bin-1]*scales[ind]))
+ outputPy.write('])\n\n')
-
+
# Canvas
outputPy.write(' # Creating a new Canvas\n')
dpi=80
@@ -795,7 +804,7 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames):
mylinewidth = self.main.datasets[ind].linewidth
mylinestyle = LineStyleType.convert2matplotlib(self.main.datasets[ind].linestyle)
- outputPy.write(' pad.hist('+\
+ outputPy.write(' n, _, _ = pad.hist('+\
'x=xData, '+\
'bins=xBinning, '+\
'weights='+myweights+',\\\n'+\
@@ -824,6 +833,10 @@ def DrawMATPLOTLIB(self,histos,scales,ref,irelhisto,filenamePy,outputnames):
' orientation="vertical")\n\n')
outputPy.write('\n')
+
+ # error bar
+ outputPy.write(' plt.errorbar(xData, n, yerr = err, fmt=\'r.\')\n')
+
# Label
outputPy.write(' # Axis\n')
outputPy.write(" plt.rc('text',usetex=False)\n")
diff --git a/madanalysis/system/architecture_info.py b/madanalysis/system/architecture_info.py
index 53b363b8..640a82e3 100644
--- a/madanalysis/system/architecture_info.py
+++ b/madanalysis/system/architecture_info.py
@@ -47,6 +47,10 @@ def __init__(self):
self.has_delphes = False
self.has_delphesMA5tune = False
+ # Flag for SQlite3
+ self.has_sqlite3 = False
+
+
# Library to put before all the others?
self.root_priority = False
self.zlib_priority = False
@@ -98,6 +102,7 @@ def __init__(self):
self.delphesMA5tune_lib=""
self.fastjet_bin_path=""
self.fastjet_lib_paths=[]
+
# C++ compiler versions
self.cpp11 = False
diff --git a/madanalysis/system/checkup.py b/madanalysis/system/checkup.py
index 4629f6f7..1c788c18 100644
--- a/madanalysis/system/checkup.py
+++ b/madanalysis/system/checkup.py
@@ -328,6 +328,8 @@ def CheckOptionalProcessingPackages(self):
return False
if not self.checker.Execute('root'):
return False
+ if not self.checker.Execute('sqlite'):
+ return False
self.archi_info.has_delphes = checker2.checkDelphes()
self.archi_info.has_delphesMA5tune = checker2.checkDelphesMA5tune()
diff --git a/madanalysis/system/detect_manager.py b/madanalysis/system/detect_manager.py
index 318399bf..aadcf1e4 100644
--- a/madanalysis/system/detect_manager.py
+++ b/madanalysis/system/detect_manager.py
@@ -126,6 +126,9 @@ def Execute(self, rawpackage):
elif package=='latex':
from madanalysis.system.detect_latex import DetectLatex
checker=DetectLatex(self.archi_info, self.user_info, self.session_info, self.debug)
+ elif package=='sqlite':
+ from madanalysis.system.detect_sqlite import DetectSQLite
+ checker=DetectSQLite(self.archi_info, self.user_info, self.session_info, self.debug)
else:
self.logger.error('the package "'+rawpackage+'" is unknown')
return False
diff --git a/madanalysis/system/detect_sqlite.py b/madanalysis/system/detect_sqlite.py
new file mode 100644
index 00000000..d74e85a6
--- /dev/null
+++ b/madanalysis/system/detect_sqlite.py
@@ -0,0 +1,96 @@
+################################################################################
+#
+# Copyright (C) 2012-2022 Jack Araz, Eric Conte & Benjamin Fuks
+# The MadAnalysis development team, email:
+#
+# This file is part of MadAnalysis 5.
+# Official website:
+#
+# MadAnalysis 5 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# MadAnalysis 5 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with MadAnalysis 5. If not, see
+#
+################################################################################
+
+
+from __future__ import absolute_import
+import logging
+import glob
+import os
+import sys
+import re
+import platform
+from shell_command import ShellCommand
+from madanalysis.enumeration.detect_status_type import DetectStatusType
+
+
+class DetectSQLite:
+
+ def __init__(self, archi_info, user_info, session_info, debug):
+ # mandatory options
+ self.archi_info = archi_info
+ self.user_info = user_info
+ self.session_info = session_info
+ self.debug = debug
+ self.name = 'SQLite3'
+ self.mandatory = False
+ self.log = []
+ self.logger = logging.getLogger('MA5')
+
+ # adding what you want here
+ self.headernames=['DatabaseManager.h']
+
+ def PrintDisableMessage(self):
+ self.logger.warning("sqlite3 not detected, multiweight output format will not be supported!")
+
+
+ def IsItVetoed(self):
+ if self.user_info.sqlite_veto:
+ self.logger.debug("user setting: veto on SQLite3")
+ return True
+ else:
+ self.logger.debug("no user veto")
+ return False
+
+
+ def AutoDetection(self):
+ # Which
+ result = ShellCommand.Which('sqlite3',all=False,mute=True)
+ if len(result)==0:
+ return DetectStatusType.UNFOUND,''
+ if self.debug:
+ self.logger.debug(" which: " + str(result[0]))
+
+ # Ok
+ return DetectStatusType.FOUND,''
+
+
+ def ExtractInfo(self):
+ # Which all
+ if self.debug:
+ result = ShellCommand.Which('sqlite3',all=True,mute=True)
+ if len(result)==0:
+ return False
+ self.logger.debug(" which-all: ")
+ for file in result:
+ self.logger.debug(" - "+str(file))
+
+ # Ok
+ return True
+
+
+ def SaveInfo(self):
+ self.archi_info.has_sqlite3 = True
+ return True
+
+
+
diff --git a/madanalysis/system/user_info.py b/madanalysis/system/user_info.py
index b13da86d..5d985040 100644
--- a/madanalysis/system/user_info.py
+++ b/madanalysis/system/user_info.py
@@ -88,6 +88,8 @@ def __init__(self):
# dvipdf
self.dvipdf_veto = None
+ self.sqlite_veto = None
+
# logger
self.logger = logging.getLogger('MA5')
diff --git a/requirements.txt b/requirements.txt
index 352d7e20..f7820720 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ scipy>=1.7.1
numpy>=1.19.5
pyhf==0.6.3
lxml>=4.6.2
+
diff --git a/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h
new file mode 100644
index 00000000..b48076d3
--- /dev/null
+++ b/tools/SampleAnalyzer/Commons/Base/DatabaseManager.h
@@ -0,0 +1,52 @@
+#pragma once
+
+
+#include
+
+class SQLiteBase;
+
+class DatabaseManager {
+ private:
+
+ SQLiteBase *manager;
+
+ public:
+ DatabaseManager(std::string path);
+
+ ~DatabaseManager();
+
+ void createCutflowTables();
+
+ void createHistoTables();
+
+ void createWeightNamesTable();
+
+ void addWeightDefinition(int id, std::string def);
+
+ void addHisto(std::string name, int bins, double xmin, double xmax, std::string regions);
+
+ void addStatistic(std::string name,
+ int id,
+ int pos_events,
+ int neg_events,
+ double pos_sum_events,
+ double neg_sum_events,
+ int pos_entries,
+ int neg_entries,
+ double pos_sum_entries,
+ double neg_sum_entries,
+ double pos_sum_squared,
+ double neg_sum_squared,
+ double pos_val_weight,
+ double neg_val_weight,
+ double pos_val2_weight,
+ double neg_val2_weight);
+
+ void addData(std::string name, int id, std::string bin, double positive, double negative);
+
+ void addCut(std::string r_name, std::string c_name);
+
+ void addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum);
+
+ void closeDB();
+};
diff --git a/tools/SampleAnalyzer/Commons/Base/OutputManager.h b/tools/SampleAnalyzer/Commons/Base/OutputManager.h
new file mode 100644
index 00000000..ce288f4e
--- /dev/null
+++ b/tools/SampleAnalyzer/Commons/Base/OutputManager.h
@@ -0,0 +1,66 @@
+#pragma once
+
+#include
+#include
+#include
+
+
+#include "SampleAnalyzer/Process/Analyzer/AnalyzerBase.h"
+#include "SampleAnalyzer/Commons/DataFormat/SampleFormat.h"
+
+//include all output type interfaces here
+#include "SampleAnalyzer/Commons/Base/DatabaseManager.h"
+
+
+using namespace std;
+using namespace MA5;
+
+
+
+//output manager object initializer takes in a AnalyzerBase pointer and a vector of arrays of size 3
+//each array constains [output type name, path to histogram file, path to cutflow file]
+class OutputManager {
+ private:
+ //base pointer type OutputBase can take on any output interface, it's only purpose is to provide
+ //and execute function, new output formats require implementation of the output interface
+ vector > output_types_;
+ AnalyzerBase *analyzer_;
+ SampleFormat *samples_;
+
+ public:
+ OutputManager(vector > output_types, AnalyzerBase *analyzer, SampleFormat *samples) : output_types_(output_types), analyzer_(analyzer), samples_(samples) {}
+ //for each output type, get output object from factory and call execute on it's interface
+ void Execute() {
+
+ //implement each individual output type here
+ for(int i = 0; i < output_types_.size(); ++i){
+
+ //implementaton for SQLite
+ if(output_types_[i][0] == "sqlite"){
+ DatabaseManager cutflow(output_types_[i][1]);
+ DatabaseManager histogram(output_types_[i][2]);
+
+ histogram.createHistoTables();
+ analyzer_->Manager()->GetPlotManager()->WriteSQL(histogram);
+ samples_->mc()->WriteWeightNames(histogram);
+ histogram.closeDB();
+
+ cutflow.createCutflowTables();
+ bool addInitial = true;
+ samples_->mc()->WriteWeightNames(cutflow);
+ for(int i = 0; i < analyzer_->Manager()->Regions().size(); ++i){
+ analyzer_->Manager()->Regions()[i]->WriteSQL(cutflow, addInitial);
+
+ }
+ cutflow.closeDB();
+
+ }
+ }
+ }
+};
+
+
+
+
+
+
diff --git a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h
index 9a09bd05..67b17881 100644
--- a/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h
+++ b/tools/SampleAnalyzer/Commons/DataFormat/MCSampleFormat.h
@@ -36,7 +36,9 @@
#include "SampleAnalyzer/Commons/DataFormat/GeneratorInfo.h"
#include "SampleAnalyzer/Commons/DataFormat/MCProcessFormat.h"
#include "SampleAnalyzer/Commons/DataFormat/WeightDefinition.h"
-#include "SampleAnalyzer/Commons/Service/LogService.h"
+#include "SampleAnalyzer/Commons/Service/LogService.h"
+
+#include "SampleAnalyzer/Commons/Base/DatabaseManager.h"
namespace MA5
@@ -80,6 +82,8 @@ class MCSampleFormat
// ----------------------- multiweights ------------------------
WeightDefinition weight_definition_;
+ std::map weight_names;
+
// ----------------------- file info ---------------------------
MAfloat64 xsection_;
@@ -125,6 +129,19 @@ class MCSampleFormat
sumweight_negative_ = 0.;
}
+ //set weight names
+ void SetName(int id, std::string name) {
+ weight_names[id] = name;
+ }
+
+
+ void WriteWeightNames(DatabaseManager &db){
+ db.createWeightNamesTable();
+ for(auto &p : weight_names){
+ db.addWeightDefinition(p.first, p.second);
+ }
+ }
+
/// Accessoir to the generator type
const MA5GEN::GeneratorType* GeneratorType() const
{ return sample_generator_; }
diff --git a/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h b/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h
index 55a51c63..8b3cc431 100644
--- a/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h
+++ b/tools/SampleAnalyzer/Commons/DataFormat/WeightCollection.h
@@ -66,6 +66,13 @@ class WeightCollection
~WeightCollection()
{ }
+ //copy constructor
+ WeightCollection(const WeightCollection &rhs){
+ for(const auto &id_weights :rhs.weights_){
+ weights_[id_weights.first]=id_weights.second;
+ }
+ }
+
/// Clear all the content
void Reset()
{ weights_.clear(); }
@@ -162,6 +169,47 @@ class WeightCollection
}
}
+ //multiply operator
+ WeightCollection& operator*=(const MAfloat64 multiple){
+ for(auto &id_value : weights_){
+ id_value.second *= multiple;
+ }
+ return *this;
+ }
+
+ //add operator
+ WeightCollection& operator+=(const MAfloat64 input){
+ for(auto &id_value : weights_){
+ id_value.second += input;
+ }
+ return *this;
+ }
+
+ //subtract operator
+ WeightCollection& operator-=(const MAfloat64 input){
+ for(auto &id_value : weights_){
+ id_value.second -= input;
+ }
+ return *this;
+ }
+
+ //divide operator
+ WeightCollection& operator/=(const MAfloat64 input){
+ for(auto &id_value : weights_){
+ id_value.second /= input;
+ }
+ return *this;
+ }
+
+
+ //assignment operator
+ WeightCollection& operator=(const MAfloat64 input){
+ for(auto &id_value : weights_){
+ id_value.second = input;
+ }
+ return *this;
+ }
+
};
diff --git a/tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp b/tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp
new file mode 100644
index 00000000..4ecd5b8b
--- /dev/null
+++ b/tools/SampleAnalyzer/Interfaces/sqlite/DatabaseManager.cpp
@@ -0,0 +1,303 @@
+// Implementation of Database Manager Interface using Pointer to Implementation Design Pattern, SQLiteBase class is only implemented if SQLite package is detected by compile scripts
+
+
+
+#include
+#include
+#include
+
+
+#include "SampleAnalyzer/Commons/Base/DatabaseManager.h"
+
+using namespace std;
+
+class SQLiteBase{
+ private:
+ // Pointer to SQLite connection
+ sqlite3 *db;
+
+ // Save any error messages
+ char *zErrMsg;
+
+ // Save the result of opening the file
+ int rc;
+
+ // Saved SQL
+ string sql;
+
+ // Create a callback function
+ static int callback(void *NotUsed, int argc, char **argv, char **azColName){
+
+ // int argc: holds the number of results
+ // (array) azColName: holds each column returned
+ // (array) argv: holds each value
+
+ for(int i = 0; i < argc; i++) {
+
+ // Show column name, value, and newline
+ cout << azColName[i] << ": " << argv[i] << endl;
+
+ }
+
+ // Insert a newline
+ cout << endl;
+
+ // Return successful
+ return 0;
+
+ }
+
+
+ bool checkDBErrors(string msg) {
+ if( rc ){
+ // Show an error message
+ cout << "DB Error: " << sqlite3_errmsg(db) << " " << msg << endl;
+ return false;
+ } else {
+ cout << " success @ " << msg << endl;
+ return true;
+ }
+
+ }
+
+
+ public:
+
+ SQLiteBase(string path) {
+ // Save the result of opening the file
+ rc = sqlite3_open(path.c_str(), &db);
+ // enable foreign key constraint
+ sqlite3_exec(db, "PRAGMA foreign_keys = ON;", 0 ,0 ,0);
+ bool open_success = checkDBErrors("opening DB");
+ if(!open_success) {
+ sqlite3_close(db);
+ cout << "open DB failed!" << endl;
+ }
+ }
+
+ void createCutflowTables() {
+
+ // Save SQL to create a table
+ sql = "CREATE TABLE IF NOT EXISTS Cutflow(" \
+ "region_name TEXT NOT NULL,"\
+ "cut_name TEXT NOT NULL," \
+ "primary key (region_name, cut_name));";
+
+ // Run the SQL
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating cutflow table");
+
+ sql = "CREATE TABLE IF NOT EXISTS Weights(" \
+ "r_name TEXT NOT NULL," \
+ "c_name TEXT NOT NULL," \
+ "id INTEGER NOT NULL," \
+ "pos_entries INTEGER NOT NULL," \
+ "neg_entries INTEGER NOT NULL," \
+ "pos_sum DOUBLE NOT NULL," \
+ "neg_sum DOUBLE NOT NULL," \
+ "pos_squared_sum DOUBLE NOT NULL,"\
+ "neg_squared_sum DOUBLE NOT NULL,"\
+ "primary key (r_name, c_name, id)" \
+ "foreign key (r_name, c_name) references Cutflow(region_name, cut_name) ON DELETE CASCADE);";
+
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating weights table");
+ }
+
+
+ void createHistoTables() {
+ sql = "CREATE TABLE IF NOT EXISTS HistoDescription("\
+ "name TEXT NOT NULL,"\
+ "num_of_bins INTEGER NOT NULL,"\
+ "xmin DOUBLE NOT NULL,"\
+ "xmax DOUBLE NOT NULL,"\
+ "regions TEXT NOT NULL,"\
+ "primary key(name) );";
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating HistoDescription table");
+
+ sql = "CREATE TABLE IF NOT EXISTS Statistics("\
+ "name TEXT NOT NULL,"\
+ "id TEXT NOT NULL,"\
+ "pos_num_events INTEGER NOT NULL,"\
+ "neg_num_events INTEGER NOT NULL,"\
+ "pos_sum_event_weights_over_events DOUBLE NOT NULL,"\
+ "neg_sum_event_weights_over_events DOUBLE NOT NULL,"\
+ "pos_entries INTEGER NOT NULL,"\
+ "neg_entries INTEGER NOT NULL,"\
+ "pos_sum_event_weights_over_entries DOUBLE NOT NULL,"\
+ "neg_sum_event_weights_over_entries DOUBLE NOT NULL,"\
+ "pos_sum_squared_weights DOUBLE NOT NULL,"\
+ "neg_sum_squared_weights DOUBLE NOT NULL,"\
+ "pos_value_times_weight DOUBLE NOT NULL,"\
+ "neg_value_times_weight DOUBLE NOT NULL,"\
+ "pos_value_squared_times_weight DOUBLE NOT NULL,"\
+ "neg_value_squared_times_weight DOUBLE NOT NULL,"\
+ "primary key(name, id) );";
+
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating Statistics table");
+
+ sql = "CREATE TABLE IF NOT EXISTS Data("\
+ "name TEXT NOT NULL,"\
+ "id INTERGER NOT NULL,"\
+ "bin TEXT NOT NULL,"\
+ "positive DOUBLE NOT NULL,"\
+ "negative DOUBLE NOT NULL,"\
+ "primary key (name, id, bin) );";
+
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating Data table");
+
+ }
+
+
+ void createWeightNamesTable() {
+ sql = "CREATE TABLE IF NOT EXISTS WeightDefinition("\
+ "id INTERGER NOT NULL,"\
+ "definition TEXT NOT NULL,"\
+ "primary key (id) );";
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ checkDBErrors("creating Weight Names table");
+ }
+
+
+ void addWeightDefinition(int id, string def) {
+ sql = "INSERT INTO WeightDefinition VALUES ('" + to_string(id) + "','" + def + "')";
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0, &zErrMsg);
+ }
+
+
+ void addHisto(string name, int bins, double xmin, double xmax, string regions) {
+ sql = "INSERT INTO HistoDescription VALUES ('" + name + "'" + "," + "'" + to_string(bins) + "'" + "," + "'" + to_string(xmin) + "'" + "," + "'" + to_string(xmax) + "'" + "," + "'" + regions + "')";
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg);
+ //checkDBErrors("inserting into Histo: " + name);
+ }
+
+
+ void addStatistic(string name,
+ int id,
+ int pos_events,
+ int neg_events,
+ double pos_sum_events,
+ double neg_sum_events,
+ int pos_entries,
+ int neg_entries,
+ double pos_sum_entries,
+ double neg_sum_entries,
+ double pos_sum_squared,
+ double neg_sum_squared,
+ double pos_val_weight,
+ double neg_val_weight,
+ double pos_val2_weight,
+ double neg_val2_weight) {
+
+ sql = "INSERT INTO Statistics VALUES ('" + name + "','" + to_string(id) + "','" + to_string(pos_events) + "','" + to_string(neg_events) + "','" + to_string(pos_sum_events) + "','" + to_string(neg_sum_events) + "','" + to_string(pos_entries) + "','" + to_string(neg_entries) + "','" + to_string(pos_sum_entries) + "','" + to_string(neg_sum_entries) + "','" + to_string(pos_sum_squared) + "','" + to_string(neg_sum_squared) + "','" + to_string(pos_val_weight) + "','" + to_string(neg_val_weight) + "','" + to_string(pos_val2_weight) + "','" + to_string(neg_val2_weight) + "')";
+
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg);
+ //checkDBErrors("inserting into Statistics: " + name);
+
+ }
+
+
+ void addData(string name, int id, string bin, double positive, double negative) {
+ sql = "INSERT INTO Data VALUES ('" + name + "'" + "," + "'" + to_string(id) + "'" + "," + "'" + bin + "'" + "," + "'" + to_string(positive) + "'" + "," + "'" + to_string(negative) + "')";
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg);
+ //checkDBErrors("inserting into Data: " + name + " " + to_string(id));
+
+ }
+
+
+ void addCut(string r_name, string c_name) {
+
+ sql = "INSERT INTO Cutflow VALUES ('" + r_name + "'" + "," + "'" + c_name + "')";
+ //cout << sql << endl;
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg);
+ //checkDBErrors("inserting cutflow: " + r_name + " " + c_name);
+ }
+
+
+ void addWeight(string r_name, string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) {
+ sql = "INSERT INTO Weights VALUES ('" + r_name + "'" + ",'" + c_name + "','" + to_string(id) + "','" + to_string(pos) + "','" + to_string(neg) + "','" + to_string(pos_sum) + "','" + to_string(neg_sum) \
+ + "','" + to_string(pos_2sum) + "','" + to_string(neg_2sum) + "')";
+ //cout << sql << endl;
+ rc = sqlite3_exec(db, sql.c_str(), callback, 0 , &zErrMsg);
+ //checkDBErrors("inserting weight values: " + r_name + " " + c_name + " weight ID: " + to_string(id));
+ }
+
+
+ void closeDB() {
+ // Close the SQL connection
+ sqlite3_close(db);
+ }
+
+};
+
+
+
+
+DatabaseManager::DatabaseManager(string path) {
+ manager = new SQLiteBase(path);
+}
+
+DatabaseManager::~DatabaseManager() {
+ delete manager;
+}
+
+void DatabaseManager::createCutflowTables() {
+ manager->createCutflowTables();
+}
+
+void DatabaseManager::createHistoTables() {
+ manager->createHistoTables();
+}
+
+void DatabaseManager::createWeightNamesTable() {
+ manager->createWeightNamesTable();
+}
+
+void DatabaseManager::addWeightDefinition(int id, std::string def) {
+ manager->addWeightDefinition(id, def);
+}
+
+void DatabaseManager::addHisto(std::string name, int bins, double xmin, double xmax, std::string regions) {
+ manager->addHisto(name, bins, xmin, xmax, regions);
+}
+
+void DatabaseManager::addStatistic(std::string name,
+ int id,
+ int pos_events,
+ int neg_events,
+ double pos_sum_events,
+ double neg_sum_events,
+ int pos_entries,
+ int neg_entries,
+ double pos_sum_entries,
+ double neg_sum_entries,
+ double pos_sum_squared,
+ double neg_sum_squared,
+ double pos_val_weight,
+ double neg_val_weight,
+ double pos_val2_weight,
+ double neg_val2_weight) {
+ manager->addStatistic(name,id,pos_events, neg_events, pos_sum_events, neg_sum_events, pos_entries, neg_entries, pos_sum_entries, neg_sum_entries, pos_sum_squared, neg_sum_squared, pos_val_weight, neg_val_weight, pos_val2_weight, neg_val2_weight);
+ }
+
+void DatabaseManager::addData(std::string name, int id, std::string bin, double positive, double negative) {
+ manager->addData(name, id, bin, positive, negative);
+}
+
+void DatabaseManager::addCut(std::string r_name, std::string c_name) {
+ manager->addCut(r_name, c_name);
+}
+
+void DatabaseManager::addWeight(std::string r_name, std::string c_name, int id, int pos, int neg, double pos_sum, double neg_sum, double pos_2sum, double neg_2sum) {
+ manager->addWeight(r_name,c_name, id, pos, neg, pos_sum, neg_sum, pos_2sum, neg_2sum);
+}
+
+void DatabaseManager::closeDB() {
+ manager->closeDB();
+}
+
+
+
diff --git a/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp b/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp
index af133836..e4bf6f13 100644
--- a/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp
+++ b/tools/SampleAnalyzer/Process/Analyzer/MergingPlots.cpp
@@ -109,7 +109,7 @@ MAbool MergingPlots::Execute(SampleFormat& mySample, const EventFormat& myEvent)
WARNING << "Found one event with a zero weight. Skipping..." << endmsg;
return false;
}
- Manager()->InitializeForNewEvent(myEventWeight);
+ Manager()->InitializeForNewEvent(myEventWeight, myEvent.mc()->multiweights().GetWeights());
// Getting number of extra jets in the event
MAuint32 njets = 0;
diff --git a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp
index 19d05390..4a34699e 100644
--- a/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp
+++ b/tools/SampleAnalyzer/Process/Core/SampleAnalyzer.cpp
@@ -39,6 +39,10 @@
#include "SampleAnalyzer/Commons/Service/ExceptionService.h"
#include "SampleAnalyzer/Commons/Service/Physics.h"
+//#include "SampleAnalyzer/Commons/Base/DatabaseManager.h"
+#include "SampleAnalyzer/Commons/Base/OutputManager.h"
+
+
using namespace MA5;
@@ -804,7 +808,7 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples,
out.Finalize();
}
-
+
// Saving the cut flows
for(MAuint32 i=0; i& mySamples,
RegionSelection *myRS = myanalysis->Manager()->Regions()[j];
std::string safname = myanalysis->Output() + "/Cutflows/" +
CleanName(myRS->GetName()) + ".saf";
- out.Initialize(&cfg_, safname.c_str());
+ out.Initialize(&cfg_, safname.c_str());
out.WriteHeader();
myRS->WriteCutflow(out);
out.WriteFoot();
@@ -822,6 +826,50 @@ MAbool SampleAnalyzer::Finalize(std::vector& mySamples,
}
}
+
+ for(MAuint32 i = 0; i < analyzers_.size(); ++i){
+
+ std::string histo_path = analyzers_[i]->Output() + "/Histograms/histo.db";
+ std::string cutflow_path = analyzers_[i]->Output() + "/Cutflows/cutflows.db";
+ vector > outputs;
+ outputs.push_back({"sqlite", cutflow_path, histo_path});
+ OutputManager output_manager(outputs, analyzers_[i], &mySamples[0]);
+ output_manager.Execute();
+ }
+
+
+
+ /*
+ // Create histo SQlite file
+ for(MAuint32 i = 0; i < analyzers_.size(); ++i){
+ std::string path = analyzers_[i]->Output() + "/Histograms/histo.db";
+ DatabaseManager dbManager(path);
+ dbManager.createHistoTables();
+ analyzers_[i]->Manager()->GetPlotManager()->WriteSQL(dbManager);
+ dbManager.closeDB();
+ }
+
+ //save multi-weight cutflows to SQL - Kyle Fan
+ for(int i = 0; i < analyzers_.size(); ++i){
+ std::string path = analyzers_[i]->Output() + "/Cutflows/cutflows.db";
+ DatabaseManager dbManager(path);
+ dbManager.createCutflowTables();
+ bool addInitial = true;
+
+ AnalyzerBase* myanalysis = analyzers_[i];
+ mySamples[0].mc()->WriteWeightNames(dbManager);
+ //insert region,cut pair to cutflow table and region,cut,weight_id (weight data) to weights table
+ for(int j = 0; j < myanalysis->Manager()->Regions().size(); ++j){
+ RegionSelection *myRS = myanalysis->Manager()->Regions()[j];
+ myRS->WriteSQL(dbManager, addInitial);
+ }
+ dbManager.closeDB();
+ }
+
+ //end of multi-weight cutflow code
+ */
+
+
// The user-defined stuff
for(MAuint32 i=0; iFinalize(summary,mySamples);
@@ -1316,4 +1364,4 @@ void SampleAnalyzer::AddDefaultInvisible()
PHYSICS->mcConfig().AddInvisibleId(16);
PHYSICS->mcConfig().AddInvisibleId(1000022);
PHYSICS->mcConfig().AddInvisibleId(1000039);
-}
\ No newline at end of file
+}
diff --git a/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp b/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp
index cd3bc81c..3b65f1a6 100644
--- a/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp
+++ b/tools/SampleAnalyzer/Process/Core/xdr_istream.cpp
@@ -50,8 +50,8 @@ xdr_istream& xdr_istream::operator>>(std::string &s)
MAchar* dummy = new MAchar[pad];
sb_->sgetn(dummy,pad);
- delete line;
- delete dummy;
+ delete [] line;
+ delete [] dummy;
return *this;
}
diff --git a/tools/SampleAnalyzer/Process/Counter/Counter.h b/tools/SampleAnalyzer/Process/Counter/Counter.h
index dde27d9f..a3b213ff 100644
--- a/tools/SampleAnalyzer/Process/Counter/Counter.h
+++ b/tools/SampleAnalyzer/Process/Counter/Counter.h
@@ -33,6 +33,30 @@
#include
#include
#include