Merge pull request #4 from marinang/userinterface

Add user interface and other new goodies!

Author: marinang

Diff for: .gitignore

@@ -1,2 +1,8 @@
 /EvtTypes
 *.pyc
+/._simjobs_
+__pycache__
+*.egg-info
+/build
+/bin
+/dist

Diff for: LaunchProduction.py

@@ -25,17 +25,17 @@
 	parser.add_argument('evttype',        metavar='<evttype>',       help="EvtType of the processus to generate.", type=str)
 	parser.add_argument('nevents',        metavar='<nevents>',       help="Number of events to produce.", type=int)
 	parser.add_argument('year',           metavar='<year>',          help="Year to simulate.", type=int, choices=[2011,2012,2015,2016,2017])
-	parser.add_argument('--polarity',     metavar='<polarity>',      help="Magnet conditions to simulate.", default='', choices=['MagUp','MagDown'])
+	parser.add_argument('--polarity',     metavar='<polarity>',      help="Magnet conditions to simulate.", choices=['MagUp','MagDown'])
 	parser.add_argument('--simcond',      metavar='<simcond>',       help="Simulation condition.", default='Sim09c', choices=['Sim09b','Sim09c'])
-	parser.add_argument('--stripping',    metavar='<stripping>',     help="Version of the stripping.", type=str, default='')
+	parser.add_argument('--stripping',    metavar='<stripping>',     help="Version of the stripping.", type=str)
 	parser.add_argument('--turbo',                                   help="Do the Turbo step.", action='store_true')
 	parser.add_argument('--mudst',                                   help="Create a muDST output instead of DST ouptut.", action='store_true')  
 	parser.add_argument('--neventsjob',   metavar='<neventsjob>',    help="Number of events per job.", type=int, default=50)
 	parser.add_argument('--runnumber',    metavar='<runnumber>',     help="Run number for Gauss.", type=int, default=baserunnumber())
 	parser.add_argument('--decfiles',     metavar='<decfiles>',      help="Version of the DecFiles package.", type=str, default='v30r5')
 	parser.add_argument('--infiles',      metavar='<infiles>',       help="External files to provide for generation, i.e LHE or HepMC files.", type=str, default='')
 
-	parser.add_argument('--cpu',          metavar='<cpu>',           help="Number of CPUs per simulation job.", type=int, default=4140)
+	parser.add_argument('--cpu',          metavar='<cpu>',           help="Number of CPUs per simulation job.", type=int)
 
 	#options to control slurm job submission #
 	#ideally you would run with these options in a screen session #
@@ -46,7 +46,10 @@
 	parser.add_argument('--npendingjobs', metavar='<npendingjobs>',  help="(Slurm option) Maximum number of pending jobs for the user.", type=int)
 	parser.add_argument('--nfreenodes',   metavar='<nfreenodes>',    help="(Slurm option) Number of nodes to be free of user's simulation jobs.", type=int)
 	parser.add_argument('--subtime',      metavar='<subtime>',       help="(Slurm option) Time interval when the jobs are sent.", nargs='+', type=int, default=[0, 23])
-
+	
+	#lxplus options
+	parser.add_argument('--toeos',                                   help="Move the jobs outputs to EOS when finished.", action='store_true') 
+	
 	opts = parser.parse_args()
 
 	opts = vars(opts).copy()

Diff for: MANIFEST.IN

@@ -0,0 +1,5 @@
+# Include the license ifile
+include LICENSE
+
+# Include the MANIFEST itself
+include MANIFEST.in

Diff for: MoveJobs.py

@@ -5,12 +5,29 @@
 from pyparsing import *
 import shutil
 import time
+from subprocess import Popen, PIPE
 
 jobdir = os.getenv("SIMOUTPUT")
 if jobdir is None :
 	jobdir = os.getenv("HOME")+"/SimulationJobs"
+eosdir = os.getenv("EOS_SIMOUTPUT")
+
+def silentrm( path ):
+	
+	P = Popen(['rm','-rf',path], stdout=PIPE, stderr=PIPE)
+	_, _ = P.communicate()
+	
+def MoveFinishedJobs( Options ):
 
-def MoveFinishedJobs( Path, Options ):
+	if Options["toeos"]:
+		dir_ = eosdir
+		os.system("xrdfs root://eoslhcb.cern.ch/ mkdir -p {0}/{evttype}/{year}/{simcond}/MagUp/xml".format( dir_, **Options))
+		os.system("xrdfs root://eoslhcb.cern.ch/ mkdir -p {0}/{evttype}/{year}/{simcond}/MagDown/xml".format( dir_,**Options))
+		proddir_ = jobdir
+	else:
+		dir_ = jobdir
+		os.system("mkdir -p {0}/{evttype}/{year}/{simcond}/MagUp/xml".format( dir_, **Options))
+		os.system("mkdir -p {0}/{evttype}/{year}/{simcond}/MagDown/xml".format( dir_,**Options))
 
 	Completed = []
 
@@ -38,35 +55,11 @@ def IsCompleted( DST, nEvents ):
 			return True
 		else:
 			return False
-			
-	def IsCompleted_1( Directory ):
-		
-		def getstatus( status, line ):
-			line = line.replace( status + ":", "")
-			line = line.replace( " ", "")
-			line = line.replace( "\n", "")
-			
-			if line == "True":
-				return True
-			elif line == "False":
-				return False
-			
-		status_file = "{0}/status".format( Directory )
+						
+	Path = dir_ + "/simProd_{evttype}_{simcond}/".format( **Options )
 
-		if os.path.isfile( status_file ):
-			
-			status_file = open( status_file, "r")
-			status = status_file.readlines()
-
-			submitted = getstatus("submitted", status[0])
-			running   = getstatus("running",   status[1])
-			completed = getstatus("completed", status[2])
-			failed    = getstatus("failed",    status[3])
-			
-			return submitted and not running and completed and not failed
-			
-		else:
-			return False
+	if Options["toeos"]:
+		Path_prod = proddir_ + "/simProd_{evttype}_{simcond}/".format( **Options )
 
 	number     = Word(nums+".")
 	text       = Word(alphas+nums) 
@@ -75,50 +68,87 @@ def getstatus( status, line ):
 	expr = number.setResultsName("year") + Suppress(underscore) + text.setResultsName("polarity") + Suppress(underscore) + number.setResultsName("nevents") + Suppress(text) 
 	expr += Suppress(underscore) + Suppress(Literal("s")) + text.setResultsName("stripping") + Suppress(underscore) + text.setResultsName("runnumber")
 
-	directories = glob.glob(Path + "{0}_*".format( Options.year ) )
+	directories = glob.glob(Path + "{year}_*".format( **Options ) )
 
 	for d in directories:
 
 		subjobdir = d.replace( Path, "")
 		parsed = expr.parseString(subjobdir)
 
+		if Options["toeos"]:
+			production_dir = Path_prod + subjobdir
+						
+		if len(os.listdir(d)) < 1:
+			silentrm(d)
+			if Options["toeos"]:
+				silentrm(production_dir)
+			continue
+				
 		nEvents = parsed.nevents
 
 		dst = "{0}/{1}_events.dst".format( d, nEvents )
 
 		xml = d + "/GeneratorLog.xml"
+		
+		completed = IsCompleted( dst, parsed.nevents )
+		finished  = IsFinished( d )
 
-		if ( IsCompleted( dst, parsed.nevents ) and IsFinished( d ) ) or IsCompleted_1( d ):
+		if completed and finished:
 			newdst  = "{0}evts_s{1}_{2}.dst".format( parsed.nevents, parsed.stripping, parsed.runnumber )
 			newxml  = "{0}.xml".format( parsed.runnumber )
 
 			print(dst)
 
-			shutil.move( dst, "{0}/{1}/{2}/{3}/{4}/{5}".format( jobdir, Options.evttype, Options.year, Options.simcond, parsed.polarity, newdst ) )
-			shutil.move( xml, "{0}/{1}/{2}/{3}/{4}/xml/{5}".format( jobdir, Options.evttype, Options.year, Options.simcond, parsed.polarity, newxml ) )
+			if Options["toeos"]:
+				os.system("xrdfs root://eoslhcb.cern.ch/ mv {0} {1}/{evttype}/{year}/{simcond}/{2}/{3}".format( dst, dir_, parsed.polarity, newdst,  **Options ))
+				os.system("xrdfs root://eoslhcb.cern.ch/ mv {0} {1}/{evttype}/{year}/{simcond}/{2}/xml/{3}".format( xml, dir_, parsed.polarity, newxml,  **Options ))
+				
+				if os.path.isdir(production_dir):
+					silentrm(production_dir)
+				
+			else:
+				shutil.move( dst, "{0}/{evttype}/{year}/{simcond}/{1}/{2}".format( dir_, parsed.polarity, newdst, **Options ) )
+				shutil.move( xml, "{0}/{{evttype}/{year}/{simcond}/{1}/xml/{2}".format( dir_, parsed.polarity, newxml , **Options ) )
+						
+			silentrm(d)
 
-			os.system("rm -rf " + d)
+		elif not completed and finished:
+			if os.path.isfile(dst):
+				os.remove(dst)
+			if os.path.isfile(xml):
+				os.remove(xml)
+			silentrm(d)
 
+			if Options["toeos"] and os.path.isdir(production_dir):
+				silentrm(production_dir)
 
+	if Options["toeos"]:	
+		directories = glob.glob(Path_prod + "{year}_*".format( **Options ) )
+		
+		for d in directories:
+			
+			subjobdir = d.replace( Path_prod, "")
+			parsed = expr.parseString(subjobdir)
+			
+			finished  = IsFinished( d )
+										
+			if len(os.listdir(d)) < 1 or finished:
+				silentrm(d)
+
 if __name__ == "__main__" :
 
 	parser = argparse.ArgumentParser(description='')
 
 	parser.add_argument('evttype',        metavar='<evttype>',       help="EvtType of the processus to generate.", type=str)
 	parser.add_argument('year',           metavar='<year>',          help="Year to simulate.", type=int, choices=[2011,2012,2015,2016,2017])
-	parser.add_argument('--simcond',      metavar='<simcond>',       help="Simulation condition.", default='Sim09c', choices=['Sim09b','Sim09c','Sim09c_LLP','Sim09c_HNL'])
+	parser.add_argument('--simcond',      metavar='<simcond>',       help="Simulation condition.", default='Sim09c', choices=['Sim09b','Sim09c'])
+	parser.add_argument('--toeos',                                   help="If jobs were moved to eos.", action='store_true') 
 
 	opts = parser.parse_args()
-	evttype = opts.evttype
-	simcond = opts.simcond
-	year    = opts.year
-	
-	jobspath = jobdir + "/simProd_{0}_{1}/".format( evttype, simcond )
 
-	os.system("mkdir -p {0}/{1}/{2}/{3}/MagUp/xml".format( jobdir, evttype, year, simcond))
-	os.system("mkdir -p {0}/{1}/{2}/{3}/MagDown/xml".format( jobdir, evttype, year, simcond))
+	opts = vars(opts).copy()
 
-	MoveFinishedJobs( jobspath, opts)
+	MoveFinishedJobs( opts )
 
 
 

Diff for: README.md

@@ -1,6 +1,22 @@
 # Simulation production
 
-Mini framework to send simulation jobs into lxplus or a slurm batch system (with access to cvmfs)!
+Mini framework to send LHCb simulation jobs on a batch system (LSF or Slurm)!
+
+## Installation
+
+To install the module do
+
+`$ python setup.py install (--user)`.
+
+You will be asked to enter some directories where you want to find your simulated samples.
+
+## Usage
+
+To launch the module just type `$ simprod`.
+
+<p align="center">
+<img width="600" height="200" src="https://github.com/marinang/SimulationProduction/blob/userinterface/etc/begin_interface.png">
+</p>
 
 You need to to know:
 
@@ -10,50 +26,73 @@ You need to to know:
 
 * Number of events you want to produce.
 
-Description of simulation setups can be found [here](https://github.com/marinang/SimulationProduction/tree/master/simjob/setup).
-	
-Before launching make sure to set the environnement variable _$SIMOUTPUT_ to the path of where you want to save the outputs of the jobs. This can be done using the _setup.sh_ script or adding this variable to your _.bashrc_.
+Description of simulation setups can be found [here](https://github.com/marinang/SimulationProduction/tree/master/simjob/setup). To start a new simulation job do:
+
+`$ j = SimulationJob( evttype=EVTTYPE, year=YEAR, nevents=NEVENTS)`
+
+`$ j.prepare()`
 
-The script to launch jobs is **LaunchProduction.py**, usage: python LaunchProduction.py EvtType Year #Events
+`$ j.send()`
 
-optional arguments:
+<p align="center">
+<img width="750" height="300" src="https://github.com/marinang/SimulationProduction/blob/userinterface/etc/submission.png">
+</p>
 
-* --polarity: Magnet conditions to simulate (MagUp or MagDown, default: half MagUp, half MagDown).
 
-* --neventsjob: Number of events per jobs (default: 50). 
+### Options
 
-* --runnumber: Run number for Gauss.
+Your have other options by default that you can change:
 
-* --simcond: Simulation condition (Sim09b, Sim09c, default: Sim09c).
+* polarity: Magnet conditions to simulate [MagUp or MagDown, default: half MagUp, half MagDown].
 
-* --stripping: Version of the stripping (default = '').
+* neventsjob: Number of events per jobs [default: 50]. 
 
-* --turbo: Run the Turbo step (output has never been tested).
+* runnumber: Run number for simulation in Gauss.
 
-* --mudst: Produce a muDST output.
+* simcond: Simulation condition [Sim09b, Sim09c, default: Sim09c].
 
-* --decfiles: Version of the DecFiles package (default = v30r5).
+* stripping: Version of the stripping (default = '').
 
-* --infiles: External files to provide for generation (for example LHE or HepMC files).
+* turbo: Run the Turbo step (output not test).
 
-* --cpu: Number of CPUs per simulation job.
+* mudst: Produce a muDST output.
+
+* decfiles: Version of the DecFiles package (default = v30r5)
+
+* infiles: External files to provide for generation (for example LHE or HepMC files).
+
+* cpu: Number of CPU memory (in MB) per simulation job.
 
-If you wish to modify any option related to an EvtType prior to launch submission, the **GetEvtType.py** script will copy every option file that are in _EvtType.py_ to a directory called _EvtTypes_. It takes the EvtType as argument.
+These argument are all available at instantiation of a SimulationJob but also as property, i.e:
+
+`$ j = SimulationJob( evttype=EVTTYPE, year=YEAR, nevents=NEVENTS, neventsjob=NEVENTSJOB)`
+
+is equivalent to
 
-If you wish to send jobs on a **Slurm** batch system you can add the following options for **LaunchProduction.py**.
+`$ j = SimulationJob( evttype=EVTTYPE, year=YEAR, nevents=NEVENTS)`
 
-* --time: Maximum running time per simulation job in hours.
+### Slurm options
 
-* --nsimjobs: Maximum number of simultaneous simulation jobs running.
+You have additionnal options for slurm batch system with default values designed for EPFL usage.
+
+* time: Maximum running time per simulation job in hours.
+
+* nsimjobs: Maximum number of simultaneous simulation jobs running.
 
-* --nsimuserjobs: Maximum number of simultaneous simulation jobs running for the user.
+* nsimuserjobs: Maximum number of simultaneous simulation jobs running for the user.
 
-* --nuserjobs: Maximum number of simultaneous jobs running for the user.
+* nuserjobs: Maximum number of simultaneous jobs running for the user.
 
-* --npendingjobs: Maximum number of pending jobs for the user.
+* npendingjobs: Maximum number of pending jobs for the user.
 
-* --nfreenodes: Number of nodes to be free of user's simulation jobs.
+* nfreenodes: Number of nodes to be free of user's simulation jobs.
 
-* --subtime: Time interval when the jobs are sent (e.g. 16 18 means from 4pm to 6pm).
+* subtime: Time interval when the jobs are sent (e.g. 16 18 means from 4pm to 6pm).
+
+## Monitoring
+
+Just after the lauching the program type `jobs` and you can see the status of submitted jobs:
 
-Note you would use these options in a **screen** session, usage: screen python LaunchProduction.py ....
+<p align="center">
+<img width="540" height="500" src="https://github.com/marinang/SimulationProduction/blob/userinterface/etc/monitor.png">
+</p>