Skip to content

Commit ee6ad5b

Browse files
giorgiopizzgiorgiopizz
committed
Update for JES production
1 parent 6c91db6 commit ee6ad5b

File tree

5 files changed

+85
-30
lines changed

5 files changed

+85
-30
lines changed

‎mkShapesRDF/lib/search_files.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,4 @@ def searchFilesDAS(
134134
files = self.cached_list_of_files[process]
135135

136136
files = list(map(lambda k: redirector + k, files))
137-
return files[1:]
137+
return files

‎mkShapesRDF/processor/framework/Steps_cfg.py‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
"formulasMC2018v9",
3232
"finalSnapshot_JES_18",
3333
],
34+
"outputFolder": "MCl1loose2018v9__MCCorr2018v9NoJERInHorn__l2tightOR2018v9__RDF",
3435
},
3536
"JES_18_test": {
3637
"isChain": True,
@@ -275,6 +276,13 @@
275276
includeVariations=False, splitVariations=False, storeNominals=True )",
276277
"module": "snapshot()",
277278
},
279+
# 'JESAbsolute':['JESAbsolute', 'JESAbsolute_2018'],\
280+
# 'JESBBEC1':['JESBBEC1', 'JESBBEC1_2018'],\
281+
# 'JESEC2':['JESEC2', 'JESEC2_2018'],\
282+
# 'JESHF':['JESHF', 'JESHF_2018'],\
283+
# 'JESRelative':['JESRelativeBal', 'JESRelativeSample_2018'],\
284+
# 'JESFlavorQCD':['JESFlavorQCD'],\
285+
# 'JESTotal':['JESTotal'],\
278286
"finalSnapshot_JES_18": {
279287
"isChain": False,
280288
"do4MC": True,
@@ -286,13 +294,7 @@
286294
eosPath='RPLME_EOSPATH', outputFilename='RPLME_OUTPUTFILENAME', \
287295
includeVariations=True, splitVariations=True, storeNominals=False,\
288296
outputMap={\
289-
'JESAbsolute':['JESAbsolute', 'JESAbsolute_2018'],\
290-
'JESBBEC1':['JESBBEC1', 'JESBBEC1_2018'],\
291-
'JESEC2':['JESEC2', 'JESEC2_2018'],\
292-
'JESHF':['JESHF', 'JESHF_2018'],\
293-
'JESRelative':['JESRelativeBal', 'JESRelativeSample_2018'],\
294-
'JESFlavorQCD':['JESFlavorQCD'],\
295-
'JESTotal':['JESTotal'],\
297+
'JES':['JESAbsolute', 'JESAbsolute_2018', 'JESBBEC1', 'JESBBEC1_2018', 'JESEC2', 'JESEC2_2018', 'JESRelativeBal', 'JESRelativeSample_2018', 'JESFlavorQCD', 'JESTotal']\
296298
} )",
297299
"module": "snapshot()",
298300
},

‎mkShapesRDF/processor/framework/processor.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,8 @@ def sciNot(value):
339339
self.fPy = self.fPy.replace("RPLME_FW", frameworkPath)
340340

341341
#: folderPathEos is the output folder path (not ending with ``/`` so that is possible to add suffix to the folder)
342-
folderPathEos = self.eosDir + "/" + self.prodName + "/" + self.step
342+
folderPathEos = self.eosDir + "/" + self.prodName + "/" # + self.step
343+
folderPathEos += Steps[self.step].get("outputFolder", self.step)
343344
self.fPy = self.fPy.replace("RPLME_EOSPATH", folderPathEos)
344345

345346
allSamples = []
@@ -372,7 +373,8 @@ def sciNot(value):
372373
files = self.searchFiles.searchFilesDAS(**files_cfg)
373374
else:
374375
files = self.searchFiles.searchFiles(**files_cfg)
375-
files = files[: self.limitFiles]
376+
if self.limitFiles != -1:
377+
files = files[: self.limitFiles]
376378

377379
if len(files) == 0:
378380
print("No files found for", sampleName, "and configuration", files_cfg)
@@ -415,22 +417,25 @@ def sciNot(value):
415417
allSamples.append(sampleName + "__part" + str(part))
416418

417419
fJdl = dedent(
418-
"""
420+
'''
419421
universe = vanilla
420422
executable = run.sh
421423
arguments = $(Folder)
422424
423425
should_transfer_files = YES
426+
transfer_output_files = ""
424427
transfer_input_files = $(Folder)/script.py
425428
426429
output = $(Folder)/out.txt
427430
error = $(Folder)/err.txt
428431
log = $(Folder)/log.txt
429432
430433
request_cpus = 1
434+
request_memory = 6GB
435+
request_disk = 2GB
431436
+JobFlavour = "workday"
432437
433-
queue 1 Folder in RPLME_ALLSAMPLES"""
438+
queue 1 Folder in RPLME_ALLSAMPLES'''
434439
)
435440

436441
fJdl = fJdl.replace("RPLME_ALLSAMPLES", " ".join(allSamples))

‎mkShapesRDF/processor/framework/samples/Summer20UL18_106x_nAODv9.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,4 @@
11
Samples = {}
2-
Samples["EWKZ2Jets_ZToLL_M-50_MJJ-120"] = {
3-
"nanoAOD": "/EWK_LLJJ_MLL-50_MJJ-120_TuneCP5_13TeV-madgraph-pythia8_dipole/RunIISummer20UL18NanoAODv9-106X_upgrade2018_realistic_v16_L1v1-v1/NANOAODSIM"
4-
} ##!!!!
5-
Samples["EWKZ2Jets_ZToLL_M-50_MJJ-120_herwig7_angular"] = {
6-
"nanoAOD": "/EWK_LLJJ_MLL_50_MJJ_120_TuneCP5_13TeV_madgraph_herwig7_angular/rgerosa-RunIISummer20UL18NanoAODv9_106X_upgrade2018_realistic_v11_NANOAODSIM-731836dbf6cca2314c65da54f8e0a2cf/USER",
7-
"instance": "prod/phys03",
8-
} ##!!!!
9-
Samples["EWKZ2Jets_ZToLL_M-50"] = {
10-
"nanoAOD": "/EWKZ2Jets_ZToLL_M-50_TuneCP5_withDipoleRecoil_13TeV-madgraph-pythia8/RunIISummer20UL18NanoAODv9-106X_upgrade2018_realistic_v16_L1v1-v2/NANOAODSIM"
11-
} ##!!!!
122
# --->Updated!###-----------------------------------------------##
133
Samples["WZTo3LNu"] = {
144
"nanoAOD": "/WZTo3LNu_TuneCP5_13TeV-amcatnloFXFX-pythia8/RunIISummer20UL18NanoAODv9-106X_upgrade2018_realistic_v16_L1v1-v2/NANOAODSIM"

‎mkShapesRDF/processor/scripts/mkPostProc.py‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,16 @@
55
import glob
66
import subprocess
77
import tabulate
8+
from fnmatch import fnmatch
89

910
#: condorDir is the path to use for condor submission, user might want to change it -> edit ``mkPostProc.py``
1011
condorDir = (
1112
"/".join(os.path.abspath(os.path.dirname(__file__)).split("/")[:-1]) + "/condor"
1213
)
1314

1415
#: eosDir is the path to use for eos submission, user might want to change it -> edit ``mkPostProc.py``
15-
eosDir = "/eos/cms/store/group/phys_smp/Latinos/vbfz/mkShapesRDF_nanoAOD"
16+
# eosDir = "/eos/cms/store/group/phys_smp/Latinos/vbfz/mkShapesRDF_nanoAOD" # just for tests
17+
eosDir = "/eos/cms/store/group/phys_higgs/cmshww/amassiro/HWWNano"
1618

1719
#: defaultRedirector is the redirector used to access files if the option ``--useRedirector 1`` is used, user might want to change it -> edit ``mkPostProc.py``
1820
defaultRedirector = "root://cms-xrd-global.cern.ch/"
@@ -133,6 +135,24 @@ def operationMode1Parser(parser=None):
133135
parser = argparse.ArgumentParser(add_help=False)
134136
parser1 = argparse.ArgumentParser(parents=[parser])
135137

138+
parser1.add_argument(
139+
"-T",
140+
"--selTree",
141+
type=str,
142+
help="List of samples to select, comma separated",
143+
required=False,
144+
default="",
145+
)
146+
147+
parser1.add_argument(
148+
"-E",
149+
"--excTree",
150+
type=str,
151+
help="List of samples to exclude, comma separated",
152+
required=False,
153+
default="",
154+
)
155+
136156
parser1.add_argument(
137157
"-r",
138158
"--resubmit",
@@ -162,7 +182,6 @@ def main():
162182
if opMode == 0:
163183
parser0 = operationMode0Parser(parser)
164184
args = parser0.parse_args()
165-
166185
selTree = args.selTree
167186
excTree = args.excTree
168187

@@ -207,20 +226,53 @@ def main():
207226
parser1 = operationMode1Parser(parser)
208227
args = parser1.parse_args()
209228
resubmit = args.resubmit
229+
230+
selTree = args.selTree
231+
excTree = args.excTree
232+
233+
if selTree == "":
234+
selTree = []
235+
else:
236+
selTree = [s.strip() for s in selTree.split(",")]
237+
238+
if excTree == "":
239+
excTree = []
240+
else:
241+
excTree = [s.strip() for s in excTree.split(",")]
242+
210243
print("Should check for errors")
211244

212245
folder = condorDir + "/" + prodName + "/" + step + "/"
213246

214247
folder = os.path.abspath(folder)
248+
errs = []
249+
files = []
250+
251+
if len(selTree) == 0:
252+
errs.extend(glob.glob(f"{folder}/*/err.txt"))
253+
files.extend(glob.glob(f"{folder}/*/script.py"))
254+
255+
for tree in selTree:
256+
errs.extend(glob.glob(f"{folder}/{tree}__part*/err.txt"))
257+
files.extend(glob.glob(f"{folder}/{tree}__part*/script.py"))
215258

216-
errs = glob.glob(f"{folder}/*/err.txt")
217-
files = glob.glob(f"{folder}/*/script.py")
259+
for tree in excTree:
260+
errs = list(
261+
filter(
262+
lambda k: not fnmatch(k, f"{folder}/{tree}__part*/err.txt"), errs
263+
)
264+
)
265+
files = list(
266+
filter(
267+
lambda k: not fnmatch(k, f"{folder}/{tree}__part*/script.py"), files
268+
)
269+
)
218270

219271
errsD = list(map(lambda k: "/".join(k.split("/")[:-1]), errs))
220272
filesD = list(map(lambda k: "/".join(k.split("/")[:-1]), files))
221273
# print(files)
222274
notFinished = list(set(filesD).difference(set(errsD)))
223-
print(notFinished)
275+
print("Waiting for\n", "\n".join(notFinished), sep="")
224276
tabulated = []
225277
tabulated.append(["Total jobs", "Finished jobs", "Running jobs"])
226278
tabulated.append([len(files), len(errs), len(notFinished)])
@@ -233,6 +285,11 @@ def main():
233285
Warning in <TClass::Init>: no dictionary for class edm::Hash<1> is available
234286
Warning in <TClass::Init>: no dictionary for class pair<edm::Hash<1>,edm::ParameterSetBlob> is available
235287
Warning in <TInterpreter::ReadRootmapFile>: class podio::
288+
TClass::Init:0: RuntimeWarning: no dictionary for class edm::Hash<1> is available
289+
TClass::Init:0: RuntimeWarning: no dictionary for class edm::ProcessHistory is available
290+
TClass::Init:0: RuntimeWarning: no dictionary for class edm::ProcessConfiguration is available
291+
TClass::Init:0: RuntimeWarning: no dictionary for class edm::ParameterSetBlob is available
292+
TClass::Init:0: RuntimeWarning: no dictionary for class pair<edm::Hash<1>,edm::ParameterSetBlob> is available
236293
real
237294
user
238295
sys
@@ -260,14 +317,15 @@ def normalErrsF(k):
260317
txt = lines.split("\n")
261318
# txt = list(filter(lambda k: k not in normalErrs, txt))
262319
txt = list(filter(lambda k: not normalErrsF(k), txt))
263-
txt = list(filter(lambda k: k.strip() != "", txt))
320+
txt = list(filter(lambda k: k != "", txt))
264321
if len(txt) > 0:
265322
print("Found unusual error in")
266323
print(err)
267-
print("\n")
268-
# print("\n".join(txt))
324+
print("\nFirst 5 lines of error file:")
325+
print("\n".join(txt[:5]))
269326
print("\n\n")
270327
toResubmit.append(err)
328+
271329
toResubmit = list(map(lambda k: "".join(k.split("/")[-2]), toResubmit))
272330
print(toResubmit)
273331
if len(toResubmit) > 0:

0 commit comments

Comments
 (0)