From 9b190bf90d34b4bf0ae31e5c000cf7f23e8f8144 Mon Sep 17 00:00:00 2001 From: Doug Burke Date: Fri, 13 Oct 2023 11:19:46 -0400 Subject: [PATCH 1/2] paramio update for CIAO 4.16 We can now access all the parameter information we need with paramio (even if a bit messy), so switch to that rather than manually parsing the parameter files. The results aren't identical, but should be "semantically" - e.g. numeric values are not guaranteed to have the same textual form but should have the samve value (e.g. 0.5e-4 vs 5e-5 and 1.0 vs 1). --- mk_runtool.py | 148 ++++++++++++++++++++++---------------------------- 1 file changed, 64 insertions(+), 84 deletions(-) diff --git a/mk_runtool.py b/mk_runtool.py index 9db8f592..5f2e63e1 100755 --- a/mk_runtool.py +++ b/mk_runtool.py @@ -43,12 +43,18 @@ """ +import glob import os import os.path import re import subprocess import sys -import glob +from typing import Any + +import paramio as pio # type: ignore + + +FuncInfo = tuple[str, str, bool] MODULE_HEADER = "mk_runtool.header" @@ -83,11 +89,7 @@ class Param: - """A parameter from a tool, as parsed from an input line. - - The constructor will throw an IOError if there was a problem - parsing the line. - """ + """A parameter from a tool.""" python_type_map = { "s": "string", @@ -96,65 +98,42 @@ class Param: "r": "number", "b": "bool"} - def __init__(self, toolname, txt): - "Parse a parameter line (txt) for the given tool." + # Typing of the return value of paramio.paramopen is tricky. + # + def __init__(self, fh: Any, toolname: str, parname: str) -> None: self.toolname = toolname - self.input_line = txt - - # Can not just split on "," since the default value may contain - # commas - e.g. stdlev1 of acis_process_events - # - # At present assume no need to handle \ for protecting string - # characters. - # - toks = [""] - in_string = False - expect_comma = False - for c in txt: - - if expect_comma: - if c == ",": - expect_comma = False - else: - raise IOError(f"Expected a comma after a quote, found {c} in [{txt}], tool={toolname}") - - # Assume that only " is used for start/end of a string - if c == '"': - if in_string: - in_string = False - expect_comma = True - else: - in_string = True - continue + self.name = parname - if c == "," and not in_string: - toks.append("") - continue - - toks[-1] += c + self.type = pio.pget(fh, f"{parname}.p_type") + self.python_type = self.python_type_map[self.type] + self.mode = pio.pget(fh, f"{parname}.p_mode") + # The value is before any expansion or redirection + self.value = pio.pget(fh, f"{parname}.p_value") - nt = len(toks) - if nt < 4: - print(f"DBG: len toks = {nt}") - print(f"DBG: toks = {toks}") - raise IOError(f"Unable to process parameter line '{txt}' for {toolname}") + # The paramio module is quite chatty when it fails. However + # we can not use contextlib,redirect_stderr to hide this + # from the user so, for now, we are stuck with it. + # + try: + self.minval = pio.pget(fh, f"{parname}.p_min") + except ValueError: + self.minval = "" - elif nt < 7: - toks.extend(["" for i in range(nt, 7)]) + try: + self.maxval = pio.pget(fh, f"{parname}.p_max") + except ValueError: + self.maxval = "" - self.name = toks[0] - self.type = toks[1] - self.python_type = self.python_type_map[self.type] - self.mode = toks[2] - self.value = toks[3] - self.minval = toks[4] - self.maxval = toks[5] - self.info = toks[6].strip() + try: + self.info = pio.pget(fh, f"{parname}.p_prompt").strip() + except ValueError: + self.info = "" # Use a case-insensitive check + # if self.name.lower() in language_keywords_lower: - print(f"WARNING: [{self.toolname}] parameter name clashes with Python reserved word:\n {txt}") + print(f"WARNING: [{self.toolname}] parameter name clashes with Python reserved word:\n {self.name}") # This is just a warning as the ParameterInfo object does the # remapping. @@ -165,12 +144,11 @@ def __init__(self, toolname, txt): if re.match(identifier, self.name) is None: nname = self.name.replace("-", "_") if re.match(identifier, nname) is None: - print(f"WARNING: [{self.toolname}] parameter name is not a valid Python identifier and does not transform '-' -> '_': {txt}") + print(f"WARNING: [{self.toolname}] parameter name is not a valid Python identifier and does not transform '-' -> '_': {self.name}") self.skip = True return # will this work? - else: - print(f"Note: converting {self.name} -> {nname}") + print(f"Note: converting {self.name} -> {nname}") if "INDEF" in self.info: self.info = self.info.replace("INDEF", "None") @@ -179,7 +157,7 @@ def __init__(self, toolname, txt): # are noted as a reminder. # if "INDEF" in [self.minval, self.maxval]: - print(f"Note: minval/maxval is INDEF in\n {txt}") + print(f"Note: minval/maxval is INDEF in\n {self.toolname}.{self.name}") # We hide the complexity of the mode (e.g. values of # h or hl or l) and just make this binary distinction @@ -223,13 +201,10 @@ def __init__(self, toolname, txt): if self.skip and self.required: raise IOError("Found a required parameter that is to be skipped!") - def __repr__(self): - return f"{self.__class__.__name__}('{self.toolname}', '{self.input_line}')" - - def __str__(self): - return self.input_line + def __str__(self) -> str: + return self.describe("") - def describe(self, sep): + def describe(self, sep: str) -> str: """Return a string of a tuple describing this parameter in the format needed by the ParameterInfo object. @@ -277,27 +252,24 @@ def describe(self, sep): return f"{sep}{ptype}({args})" -def get_param_list(dirname, toolname): +def get_param_list(dirname: str, + toolname: str) -> tuple[list[Param], list[Param]]: """Return a tuple of required and parameter values, where each entry is a list (which can be empty). Each list entry is a dictionary containing information on that parameter. """ - # Unfortunately the Python paramio module does not provide - # all of the functionality of the S-Lang version (e.g. - # plist_names) so we have to parse the parameter file + # Unfortunately the paramio interface can be a bit noisy. # pname = os.path.join(dirname, f"{toolname}.par") - with open(pname, "r") as pf: + fh = pio.paramopen(pname, "rH") + try: req_params = [] opt_params = [] - for l in pf.readlines(): - l = l.strip() - if len(l) == 0 or l.startswith("#"): - continue - p = Param(toolname, l) + for pname in pio.plist(fh): + p = Param(fh, toolname, pname) if p.skip: continue @@ -306,10 +278,15 @@ def get_param_list(dirname, toolname): else: opt_params.append(p) + finally: + pio.paramclose(fh) + return (req_params, opt_params) -def create_output(dirname, toolname, istool=True): +def create_output(dirname: str, + toolname: str, + istool: bool = True) -> str: """Return Python code for the given tool. Parameters @@ -345,7 +322,8 @@ def create_output(dirname, toolname, istool=True): return out.replace("\t", " ") -def add_output(funcinfo, parname): +def add_output(funcinfo: dict[str, FuncInfo], + parname: str) -> None: """Store information on the tool for later use. It is assumed that the "installed" version is processed @@ -390,17 +368,19 @@ def add_output(funcinfo, parname): funcinfo[toolname] = (dirname, toolname, istool) -def print_module_section(ofh, filename): +def print_module_section(ofh, filename: str) -> None: "Add contents of filename to ofh" - with open(filename, "r") as f: + with open(filename, "r", encoding="UTF-8") as f: for l in f.readlines(): ofh.write(l) ofh.flush() -def add_par_files(parinfo, dirname, pardir='param'): +def add_par_files(parinfo: dict[str, FuncInfo], + dirname: str, + pardir: str = 'param') -> None: """Update parinfo with .par files in dirname. Parameters @@ -429,7 +409,7 @@ def add_par_files(parinfo, dirname, pardir='param'): add_output(parinfo, fname) -def doit(): +def doit() -> None: ascds_install = os.getenv("ASCDS_INSTALL") if ascds_install is None: @@ -455,9 +435,9 @@ def doit(): print(f"Input directores:\n {ascds_install}\n {ascds_contrib}\n") - with open(oname, "w") as ofh: + with open(oname, "w", encoding="UTF-8") as ofh: print_module_section(ofh, MODULE_HEADER) - tools = {} + tools: dict[str, FuncInfo] = {} add_par_files(tools, ascds_install) add_par_files(tools, ascds_contrib) From 0ae6a515d4440b98fa09a36acda373fce68d6224 Mon Sep 17 00:00:00 2001 From: Doug Burke Date: Fri, 13 Oct 2023 11:22:00 -0400 Subject: [PATCH 2/2] Update runtool These changes **shold only** be textual and not provide a semantic difference (e.g. 1.0 vs 1), but we need to check this assumption out. --- ciao_contrib/runtool.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ciao_contrib/runtool.py b/ciao_contrib/runtool.py index 084d4b2b..28a0f5cb 100644 --- a/ciao_contrib/runtool.py +++ b/ciao_contrib/runtool.py @@ -2530,14 +2530,14 @@ def make_tool(toolname): parinfo['acis_find_afterglow'] = { 'istool': True, 'req': [ParValue("infile","s","Name of input event-data file(s)",None),ParValue("outfile","s","Name of output bad-pixel file",None),ParValue("badpixfile","s","Name of input bad-pixel file",None),ParValue("maskfile","s","Name of input mask file",None),ParValue("statfile","s","Name of input exposure-statistics file",None)], - 'opt': [ParRange("expnowindow","i","Number of frames in the sliding time window",10,1,100),ParRange("probthresh","r","Minimum post-trials significance of potential afterglows (1 sigma = 0.159, 2 sigma = 0.0228, and 3 sigma = 0.00135)",0.001,1.0e-10,0.1),ParRange("cntthresh","i","Minimum number of events in an afterglow",4,2,10),ParRange("regwidth","i","Size of reference region (e.g., 7 pixels x 7 pixels)",7,3,255),ParRange("nfpixreg","i","Size of the pixel region for calculation of the Nominal Fluence",32,16,256),ParRange("nfrepeat","i","Number of iterations during calculation of Nominal Fluence",10,1,30),ParRange("tolerance","r","Tolerance for series calculations",1.0e-15,1.0e-16,1.0e-6),ParValue("runhotpix","b","Run the hot pixel portion of the algorithm?",True),ParValue("clobber","b","Overwrite output file if it exists?",False),ParRange("verbose","i","Amount of messages produced (0=none, 5=all)",0,0,5)], + 'opt': [ParRange("expnowindow","i","Number of frames in the sliding time window",10,1,100),ParRange("probthresh","r","Minimum post-trials significance of potential afterglows (1 sigma = 0.159, 2 sigma = 0.0228, and 3 sigma = 0.00135)",0.001,1e-10,0.1),ParRange("cntthresh","i","Minimum number of events in an afterglow",4,2,10),ParRange("regwidth","i","Size of reference region (e.g., 7 pixels x 7 pixels)",7,3,255),ParRange("nfpixreg","i","Size of the pixel region for calculation of the Nominal Fluence",32,16,256),ParRange("nfrepeat","i","Number of iterations during calculation of Nominal Fluence",10,1,30),ParRange("tolerance","r","Tolerance for series calculations",1.0e-15,1e-16,1e-06),ParValue("runhotpix","b","Run the hot pixel portion of the algorithm?",True),ParValue("clobber","b","Overwrite output file if it exists?",False),ParRange("verbose","i","Amount of messages produced (0=none, 5=all)",0,0,5)], } parinfo['acis_process_events'] = { 'istool': True, 'req': [ParValue("infile","f","Input event file or stack",None),ParValue("outfile","f","Output event file name",None),ParValue("acaofffile","f","aspect offset file ( NONE | none | )",'NONE')], - 'opt': [ParValue("apply_cti","b","Apply CTI adjustment?",True),ParValue("apply_tgain","b","Apply time-dependent gain adjustment?",True),ParValue("obsfile","f","obs.par file for output file keywords ( NONE | none | )",'NONE'),ParValue("geompar","f","Parameter file for Pixlib Geometry files",'geom'),ParValue("logfile","f","debug log file ( STDOUT | stdout | )",'stdout'),ParValue("gradefile","f","grade mapping file ( NONE | none | CALDB | )",'CALDB'),ParValue("grade_image_file","f","grade image file for cti correcting graded mode ( NONE | none | CALDB | )",'CALDB'),ParValue("gainfile","f","acis gain file ( NONE | none | CALDB | )",'CALDB'),ParValue("badpixfile","f","acis bad pixel file ( NONE | none | )",'NONE'),ParValue("threshfile","f","split threshold file ( NONE | none | CALDB | )",'CALDB'),ParValue("ctifile","f","acis CTI file ( NONE | none | CALDB | )",'CALDB'),ParValue("tgainfile","f","gain adjustment file ( NONE | none | CALDB | )",'CALDB'),ParValue("mtlfile","s","Mission time line file with FP_TEMP data",'NONE'),ParValue("eventdef","s","output format definition",')stdlev1'),ParValue("doevtgrade","b","Determine event flight grade?",True),ParValue("check_vf_pha","b","Check very faint pixels?",False),ParRange("trail","r","Trail fraction",0.027,0,1),ParValue("calculate_pi","b","perform pha->pi conversion? (requires gain file)",True),ParRange("pi_bin_width","r","Width of Pi bin in eV",14.6,1.0,100.0),ParRange("pi_num_bins","i","Number of values to bin energy into",1024,256,32767),ParRange("max_cti_iter","i","Maximum iterations for the CTI adjustment of each event",15,1,20),ParRange("cti_converge","r","The convergence criterion for each CTI-adjusted pixel in adu",0.1,0.1,1),ParValue("clobber","b","Overwrite output event file if it already exists?",False),ParRange("verbose","i","level of debug detail (0=none, 5=most)",0,0,5),ParSet("stop","s","where to end transformations",'sky',["chip","tdet","det","tan","sky","none"]),ParRange("rand_seed","i","random seed (for pixlib), 0 = use time dependent seed",1,0,None),ParValue("rand_pha","b","Randomize the pha value used in gain calculations",True),ParSet("pix_adj","s","Sub-pixel adjustment algorithm",'EDSER',["EDSER","CENTROID","RANDOMIZE","NONE"]),ParValue("subpixfile","f","Name of input sub-pixel calibration file",'CALDB'),ParValue("stdlev1","s","TE faint modes event definition string",'{d:time,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("grdlev1","s","TE graded event format definition string",'{d:time,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("cclev1","s","CC faint event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,f:sky_1d,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("ccgrdlev1","s","cc graded event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,f:sky_1d,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("cclev1a","s","Lev1.5 CC faint event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,f:chipy_tg,f:chipy_zo,s:tdet,f:det,f:sky,f:sky_1d,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,f:rd,s:tg_m,f:tg_lam,f:tg_mlam,s:tg_srcid,s:tg_part,s:tg_smap,x:status}'),ParValue("ccgrdlev1a","s","lev1.5 cc graded event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,f:chipy_tg,f:chipy_zo,s:tdet,f:det,f:sky,f:sky_1d,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,f:rd,s:tg_m,f:tg_lam,f:tg_mlam,s:tg_srcid,s:tg_part,s:tg_smap,x:status}')], + 'opt': [ParValue("apply_cti","b","Apply CTI adjustment?",True),ParValue("apply_tgain","b","Apply time-dependent gain adjustment?",True),ParValue("obsfile","f","obs.par file for output file keywords ( NONE | none | )",'NONE'),ParValue("geompar","f","Parameter file for Pixlib Geometry files",'geom'),ParValue("logfile","f","debug log file ( STDOUT | stdout | )",'stdout'),ParValue("gradefile","f","grade mapping file ( NONE | none | CALDB | )",'CALDB'),ParValue("grade_image_file","f","grade image file for cti correcting graded mode ( NONE | none | CALDB | )",'CALDB'),ParValue("gainfile","f","acis gain file ( NONE | none | CALDB | )",'CALDB'),ParValue("badpixfile","f","acis bad pixel file ( NONE | none | )",'NONE'),ParValue("threshfile","f","split threshold file ( NONE | none | CALDB | )",'CALDB'),ParValue("ctifile","f","acis CTI file ( NONE | none | CALDB | )",'CALDB'),ParValue("tgainfile","f","gain adjustment file ( NONE | none | CALDB | )",'CALDB'),ParValue("mtlfile","s","Mission time line file with FP_TEMP data",'NONE'),ParValue("eventdef","s","output format definition",')stdlev1'),ParValue("doevtgrade","b","Determine event flight grade?",True),ParValue("check_vf_pha","b","Check very faint pixels?",False),ParRange("trail","r","Trail fraction",0.027,0,1),ParValue("calculate_pi","b","perform pha->pi conversion? (requires gain file)",True),ParRange("pi_bin_width","r","Width of Pi bin in eV",14.6,1,100),ParRange("pi_num_bins","i","Number of values to bin energy into",1024,256,32767),ParRange("max_cti_iter","i","Maximum iterations for the CTI adjustment of each event",15,1,20),ParRange("cti_converge","r","The convergence criterion for each CTI-adjusted pixel in adu",0.1,0.1,1),ParValue("clobber","b","Overwrite output event file if it already exists?",False),ParRange("verbose","i","level of debug detail (0=none, 5=most)",0,0,5),ParSet("stop","s","where to end transformations",'sky',["chip","tdet","det","tan","sky","none"]),ParRange("rand_seed","i","random seed (for pixlib), 0 = use time dependent seed",1,0,None),ParValue("rand_pha","b","Randomize the pha value used in gain calculations",True),ParSet("pix_adj","s","Sub-pixel adjustment algorithm",'EDSER',["EDSER","CENTROID","RANDOMIZE","NONE"]),ParValue("subpixfile","f","Name of input sub-pixel calibration file",'CALDB'),ParValue("stdlev1","s","TE faint modes event definition string",'{d:time,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("grdlev1","s","TE graded event format definition string",'{d:time,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("cclev1","s","CC faint event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,f:sky_1d,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("ccgrdlev1","s","cc graded event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,s:tdet,f:det,f:sky,f:sky_1d,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,x:status}'),ParValue("cclev1a","s","Lev1.5 CC faint event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,f:chipy_tg,f:chipy_zo,s:tdet,f:det,f:sky,f:sky_1d,s:phas,l:pha,l:pha_ro,f:energy,l:pi,s:fltgrade,s:grade,f:rd,s:tg_m,f:tg_lam,f:tg_mlam,s:tg_srcid,s:tg_part,s:tg_smap,x:status}'),ParValue("ccgrdlev1a","s","lev1.5 cc graded event format definition string",'{d:time,d:time_ro,l:expno,s:ccd_id,s:node_id,s:chip,f:chipy_tg,f:chipy_zo,s:tdet,f:det,f:sky,f:sky_1d,l:pha,l:pha_ro,s:corn_pha,f:energy,l:pi,s:fltgrade,s:grade,f:rd,s:tg_m,f:tg_lam,f:tg_mlam,s:tg_srcid,s:tg_part,s:tg_smap,x:status}')], } @@ -2585,7 +2585,7 @@ def make_tool(toolname): parinfo['aplimits'] = { 'istool': True, - 'req': [ParRange("prob_false_detection","r","Upper limit on the probability of a type I error",.1,0., 1.),ParRange("prob_missed_detection","r","Upper limit on the probability of a type II error",.5,0., 1.),ParValue("outfile","f","Filename of output file",None),ParRange("T_s","r","Exposure time in source aperture",1,0,None),ParRange("A_s","r","Geometric area of source aperture",1,0,None),ParRange("bkg_rate","r","Background count rate",None,0,None),ParRange("m","i","Number of counts in background aperture",None,0,None),ParRange("T_b","r","Exposure time in background aperture",1,0,None),ParRange("A_b","r","Geometric area of background aperture",1,0,None)], + 'req': [ParRange("prob_false_detection","r","Upper limit on the probability of a type I error",.1,0,1),ParRange("prob_missed_detection","r","Upper limit on the probability of a type II error",.5,0,1),ParValue("outfile","f","Filename of output file",None),ParRange("T_s","r","Exposure time in source aperture",1,0,None),ParRange("A_s","r","Geometric area of source aperture",1,0,None),ParRange("bkg_rate","r","Background count rate",None,0,None),ParRange("m","i","Number of counts in background aperture",None,0,None),ParRange("T_b","r","Exposure time in background aperture",1,0,None),ParRange("A_b","r","Geometric area of background aperture",1,0,None)], 'opt': [ParRange("max_counts","i","Background count number above which the uncertainty on the background is ignored",50,0,None),ParRange("maxfev","i","Maximal number of function evaluations in numerical root finding",500,0,None),ParRange("verbose","i","Debug Level(0-5)",1,0,5),ParValue("clobber","b","OK to overwrite existing output file?",False)], } @@ -2760,7 +2760,7 @@ def make_tool(toolname): parinfo['csmooth'] = { 'istool': True, - 'req': [ParValue("infile","f","input file name",None),ParValue("sclmap","f","image of user-supplied map of smoothing scales",None),ParValue("outfile","f","output file name",None),ParValue("outsigfile","f","output significance image",'.'),ParValue("outsclfile","f","output scales [kernel sizes] image",'.'),ParSet("conmeth","s","Convolution method.",'fft',["slide","fft"]),ParSet("conkerneltype","s","Convolution kernel type.",'gauss',["gauss","tophat"]),ParValue("sigmin","r","minimal significance, S/N ratio",4),ParRange("sigmax","r","maximal significance, S/N ratio",5,")sigmin",None),ParValue("sclmin","r","initial (minimal) smoothing scale [pixel]",None),ParRange("sclmax","r","maximal smoothing scale [pixel]",None,")sclmin",None),ParSet("sclmode","s","compute smoothing scales or user user-supplied map",'compute',["compute","user"])], + 'req': [ParValue("infile","f","input file name",None),ParValue("sclmap","f","image of user-supplied map of smoothing scales",None),ParValue("outfile","f","output file name",None),ParValue("outsigfile","f","output significance image",'.'),ParValue("outsclfile","f","output scales [kernel sizes] image",'.'),ParSet("conmeth","s","Convolution method.",'fft',["slide","fft"]),ParSet("conkerneltype","s","Convolution kernel type.",'gauss',["gauss","tophat"]),ParValue("sigmin","r","minimal significance, S/N ratio",4),ParRange("sigmax","r","maximal significance, S/N ratio",5,4,None),ParValue("sclmin","r","initial (minimal) smoothing scale [pixel]",None),ParRange("sclmax","r","maximal smoothing scale [pixel]",None,"INDEF",None),ParSet("sclmode","s","compute smoothing scales or user user-supplied map",'compute',["compute","user"])], 'opt': [ParValue("stepzero","r","initial stepsize",0.01),ParSet("bkgmode","s","background treatment",'local',["local","user"]),ParValue("bkgmap","f","user-supplied input background image",None),ParValue("bkgerr","f","user-supplied input background error image",None),ParValue("clobber","b","clobber existing output",False),ParRange("verbose","i","verbosity of processing comments",0,0,5)], } @@ -2782,7 +2782,7 @@ def make_tool(toolname): parinfo['destreak'] = { 'istool': True, 'req': [ParValue("infile","f","Input dataset/block specification",None),ParValue("outfile","f","Output dataset/block specification",None)], - 'opt': [ParValue("max","s","streak threshold syntax: m OR m:m:m:m",None),ParRange("max_rowloss_fraction","r","Maximum fraction of avg streaks/node/frame",5.0e-5,0.3e-4,1),ParValue("num_sigma","r","Sigma value for determining streak threshold",1.0),ParValue("filter","b","Discard tagged events",True),ParValue("mask","s","Filter to select candidate streak events",'[status=0,grade=0,2:4,6]'),ParValue("ccd_id","s","CCD ID to filter",'8'),ParValue("ccd_col","s","CCD ID column name",'ccd_id'),ParValue("node_col","s","Node ID column name ('none' for single node)",'node_id'),ParValue("exptime","r","frame time (s) (reads EXPTIME if no pos. value given)",-1),ParValue("countfile","f","filename for event row-count distribution",None),ParValue("fracfile","f","filename for cumulative streak contam function",None),ParValue("timefile","f","filename for exposure time lost per row",None),ParRange("verbose","i","Debug Level(0-5)",0,0,5),ParValue("clobber","b","Clobber existing file",False)], + 'opt': [ParValue("max","s","streak threshold syntax: m OR m:m:m:m",None),ParRange("max_rowloss_fraction","r","Maximum fraction of avg streaks/node/frame",5.0e-5,3e-05,1),ParValue("num_sigma","r","Sigma value for determining streak threshold",1.0),ParValue("filter","b","Discard tagged events",True),ParValue("mask","s","Filter to select candidate streak events",'[status=0,grade=0,2:4,6]'),ParValue("ccd_id","s","CCD ID to filter",'8'),ParValue("ccd_col","s","CCD ID column name",'ccd_id'),ParValue("node_col","s","Node ID column name ('none' for single node)",'node_id'),ParValue("exptime","r","frame time (s) (reads EXPTIME if no pos. value given)",-1),ParValue("countfile","f","filename for event row-count distribution",None),ParValue("fracfile","f","filename for cumulative streak contam function",None),ParValue("timefile","f","filename for exposure time lost per row",None),ParRange("verbose","i","Debug Level(0-5)",0,0,5),ParValue("clobber","b","Clobber existing file",False)], } @@ -3286,7 +3286,7 @@ def make_tool(toolname): parinfo['hrc_process_events'] = { 'istool': True, 'req': [ParValue("infile","f","input level 0 event file/stack",None),ParValue("outfile","f","output level 1 file",None),ParValue("badpixfile","f","bad pixel file ( NONE | none | )",'NONE'),ParValue("acaofffile","f","aspect offset file ( NONE | none | )",'NONE')], - 'opt': [ParValue("obsfile","f","obs.par file for output file keywords ( NONE | none | )",'NONE'),ParValue("geompar","f","Parameter file for Pixlib Geometry files",'geom'),ParValue("do_ratio","b","perform ratio validity checks",True),ParValue("do_amp_sf_cor","b","perform amp_sf correction (y/n) ?",True),ParValue("gainfile","f","gain correction image file ( NONE | none | )",'CALDB'),ParValue("ADCfile","f","ADC correction table file ( NONE | none | )",'NONE'),ParValue("degapfile","f","degap factors (NONE | none | COEFF | )",'CALDB'),ParValue("hypfile","f","Hyperbolic test coefficients file ( NONE | none | )",'CALDB'),ParValue("ampsfcorfile","f","caldb file for amp_sf_correction( NONE | none | )",'CALDB'),ParValue("tapfile","f","tap ring test coefficients file ( NONE | none | )",'CALDB'),ParValue("ampsatfile","f","ADC saturation test file ( NONE | none | )",'CALDB'),ParValue("evtflatfile","f","Event flatness test file ( NONE | none | )",'CALDB'),ParValue("badfile","f","output level 1 bad event file",'lev1_bad_evts.fits'),ParValue("logfile","f","debug log file (STDOUT | stdout | )",'stdout'),ParValue("eventdef","s","output format definition",')stdlev1'),ParValue("badeventdef","s","output format definition",')badlev1'),ParRange("grid_ratio","r","charge ratio",0.5,0.0,1.0),ParRange("pha_ratio","r","pha ratio",0.5,0.0,1.0),ParValue("wire_charge","i","turn on center wire test (-1=off,0=on)",0),ParValue("cfu1","r","u axis 1st order cor. factor",1.0),ParValue("cfu2","r","u axis 2nd order cor. factor",0.0),ParValue("cfv1","r","v axis 1st order cor. factor",1.0),ParValue("cfv2","r","v axis 2nd order cor. factor",0.0),ParValue("amp_gain","r","amp gain",75.0),ParRange("rand_seed","i","random seed (for pixlib), 0 = use time dependent seed",1,0,32767),ParSet("rand_pix_size","r","pixel randomization width (-size..+size), 0.0=no randomization",0.0,[0.0,0.5]),ParSet("start","s","start transformations at",'coarse',["coarse","chip","tdet"]),ParSet("stop","s","end transformations at",'sky',["none","chip","tdet","det","sky"]),ParValue("stdlev1","s","event format definition string",'{d:time,s:crsv,s:crsu,s:amp_sf,s:av1,s:av2,s:av3,s:au1,s:au2,s:au3,l:raw,s:chip,l:tdet,f:det,f:sky,f:samp,s:pha,s:pi,s:sumamps,s:chip_id,x:status}'),ParValue("badlev1","s","event format definition string",'{d:time,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,f:samp,s:pha}'),ParValue("hsilev1","s","event format definition string",'{d:time,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,s:chipx,s:chipy,s:tdetx,s:tdety,s:x,s:y,l:fpz,f:samp,s:pha,s:vstat,s:estat}'),ParValue("simlev1","s","sim event definition string",'{l:tick,i:scifr,i:mjf,s:mnf,s:evtctr,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,s:tdetx,s:tdety,f:samp,s:pha,s:vstat,s:estat}'),ParValue("fltlev1","s","event format definition string",'{d:time,s:crsv,s:crsu,s:amp_sf,s:av1,s:av2,s:av3,s:au1,s:au2,s:au3,s:chipx,s:chipy,l:tdetx,l:tdety,s:detx,s:dety,s:x,s:y,f:samp,s:pha,s:sumamps,s:chip_id,l:status}'),ParValue("clobber","b","Overwrite output event file if it already exists?",False),ParRange("verbose","i","level of debug detail (0=none, 5=most)",0,0,5)], + 'opt': [ParValue("obsfile","f","obs.par file for output file keywords ( NONE | none | )",'NONE'),ParValue("geompar","f","Parameter file for Pixlib Geometry files",'geom'),ParValue("do_ratio","b","perform ratio validity checks",True),ParValue("do_amp_sf_cor","b","perform amp_sf correction (y/n) ?",True),ParValue("gainfile","f","gain correction image file ( NONE | none | )",'CALDB'),ParValue("ADCfile","f","ADC correction table file ( NONE | none | )",'NONE'),ParValue("degapfile","f","degap factors (NONE | none | COEFF | )",'CALDB'),ParValue("hypfile","f","Hyperbolic test coefficients file ( NONE | none | )",'CALDB'),ParValue("ampsfcorfile","f","caldb file for amp_sf_correction( NONE | none | )",'CALDB'),ParValue("tapfile","f","tap ring test coefficients file ( NONE | none | )",'CALDB'),ParValue("ampsatfile","f","ADC saturation test file ( NONE | none | )",'CALDB'),ParValue("evtflatfile","f","Event flatness test file ( NONE | none | )",'CALDB'),ParValue("badfile","f","output level 1 bad event file",'lev1_bad_evts.fits'),ParValue("logfile","f","debug log file (STDOUT | stdout | )",'stdout'),ParValue("eventdef","s","output format definition",')stdlev1'),ParValue("badeventdef","s","output format definition",')badlev1'),ParRange("grid_ratio","r","charge ratio",0.5,0,1),ParRange("pha_ratio","r","pha ratio",0.5,0,1),ParValue("wire_charge","i","turn on center wire test (-1=off,0=on)",0),ParValue("cfu1","r","u axis 1st order cor. factor",1.0),ParValue("cfu2","r","u axis 2nd order cor. factor",0.0),ParValue("cfv1","r","v axis 1st order cor. factor",1.0),ParValue("cfv2","r","v axis 2nd order cor. factor",0.0),ParValue("amp_gain","r","amp gain",75.0),ParRange("rand_seed","i","random seed (for pixlib), 0 = use time dependent seed",1,0,32767),ParSet("rand_pix_size","r","pixel randomization width (-size..+size), 0.0=no randomization",0.0,[0,0.5]),ParSet("start","s","start transformations at",'coarse',["coarse","chip","tdet"]),ParSet("stop","s","end transformations at",'sky',["none","chip","tdet","det","sky"]),ParValue("stdlev1","s","event format definition string",'{d:time,s:crsv,s:crsu,s:amp_sf,s:av1,s:av2,s:av3,s:au1,s:au2,s:au3,l:raw,s:chip,l:tdet,f:det,f:sky,f:samp,s:pha,s:pi,s:sumamps,s:chip_id,x:status}'),ParValue("badlev1","s","event format definition string",'{d:time,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,f:samp,s:pha}'),ParValue("hsilev1","s","event format definition string",'{d:time,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,s:chipx,s:chipy,s:tdetx,s:tdety,s:x,s:y,l:fpz,f:samp,s:pha,s:vstat,s:estat}'),ParValue("simlev1","s","sim event definition string",'{l:tick,i:scifr,i:mjf,s:mnf,s:evtctr,s:crsu,s:crsv,s:au1,s:au2,s:au3,s:av1,s:av2,s:av3,s:tdetx,s:tdety,f:samp,s:pha,s:vstat,s:estat}'),ParValue("fltlev1","s","event format definition string",'{d:time,s:crsv,s:crsu,s:amp_sf,s:av1,s:av2,s:av3,s:au1,s:au2,s:au3,s:chipx,s:chipy,l:tdetx,l:tdety,s:detx,s:dety,s:x,s:y,f:samp,s:pha,s:sumamps,s:chip_id,l:status}'),ParValue("clobber","b","Overwrite output event file if it already exists?",False),ParRange("verbose","i","level of debug detail (0=none, 5=most)",0,0,5)], } @@ -3419,7 +3419,7 @@ def make_tool(toolname): parinfo['mkrmf'] = { 'istool': True, 'req': [ParValue("infile","f","name of FEF input file",None),ParValue("outfile","f","name of RMF output file",None),ParValue("axis1","s","axis-1(name=lo:hi:btype)",None),ParValue("axis2","s","axis-2(name=lo:hi:btype)",None)], - 'opt': [ParValue("logfile","f","name of log file",'STDOUT'),ParValue("weights","f","name of weight file",None),ParRange("thresh","r","low threshold of energy cut-off probability",1e-5,0.0,None),ParValue("outfmt","s","RMF output format (legacy|cxc)",'legacy'),ParValue("clobber","b","overwrite existing output file (yes|no)?",False),ParRange("verbose","i","verbosity level (0 = no display)",0,0,5),ParValue("axis3","s","axis-3(name=lo:hi:btype)",'none'),ParValue("axis4","s","axis-4(name=lo:hi:btype)",'none'),ParValue("axis5","s","axis-5(name=lo:hi:btype)",'none')], + 'opt': [ParValue("logfile","f","name of log file",'STDOUT'),ParValue("weights","f","name of weight file",None),ParRange("thresh","r","low threshold of energy cut-off probability",1e-5,0,None),ParValue("outfmt","s","RMF output format (legacy|cxc)",'legacy'),ParValue("clobber","b","overwrite existing output file (yes|no)?",False),ParRange("verbose","i","verbosity level (0 = no display)",0,0,5),ParValue("axis3","s","axis-3(name=lo:hi:btype)",'none'),ParValue("axis4","s","axis-4(name=lo:hi:btype)",'none'),ParValue("axis5","s","axis-5(name=lo:hi:btype)",'none')], } @@ -3496,7 +3496,7 @@ def make_tool(toolname): parinfo['psf_contour'] = { 'istool': True, 'req': [ParValue("infile","f","Input event file or image",None),ParValue("outroot","f","Output directory and root filename",None),ParValue("pos","s","Position: coordinate or input file with columns RA and DEC",None)], - 'opt': [ParSet("method","s","Region creating algorithm",'contour',["contour","lasso","fitted_ellipse","ecf_ellipse","convex_hull"]),ParRange("energy","r","Monochromatic energy to simulate PSF",1.0,0.3,10.0),ParRange("fraction","r","Target fraction of the PSF to include",0.9,0.6,0.95),ParRange("tolerance","r","Tolerance on fraction",0.01,0.0001,0.1),ParRange("flux","r","Photon flux to simulate",0.01,0.0000001,0.01),ParValue("fovfile","f","Input field of view file",None),ParValue("marx_root","f","Directory where MARX is installed",'${MARX_ROOT}'),ParValue("random_seed","i","PSF random seed, -1: current time",-1),ParValue("parallel","b","Run processes in parallel?",True),ParValue("nproc","i","Number of processors to use (None:use all available)",None),ParRange("verbose","i","Amount of tool chatter",1,0,5),ParValue("clobber","b","Delete output file if it already exists?",False)], + 'opt': [ParSet("method","s","Region creating algorithm",'contour',["contour","lasso","fitted_ellipse","ecf_ellipse","convex_hull"]),ParRange("energy","r","Monochromatic energy to simulate PSF",1.0,0.3,10),ParRange("fraction","r","Target fraction of the PSF to include",0.9,0.6,0.95),ParRange("tolerance","r","Tolerance on fraction",0.01,0.0001,0.1),ParRange("flux","r","Photon flux to simulate",0.01,1e-07,0.01),ParValue("fovfile","f","Input field of view file",None),ParValue("marx_root","f","Directory where MARX is installed",'${MARX_ROOT}'),ParValue("random_seed","i","PSF random seed, -1: current time",-1),ParValue("parallel","b","Run processes in parallel?",True),ParValue("nproc","i","Number of processors to use (None:use all available)",None),ParRange("verbose","i","Amount of tool chatter",1,0,5),ParValue("clobber","b","Delete output file if it already exists?",False)], } @@ -3510,7 +3510,7 @@ def make_tool(toolname): parinfo['psfsize_srcs'] = { 'istool': True, 'req': [ParValue("infile","f","Input image or event file",None),ParValue("pos","s","Input position: either filename or ra,dec",None),ParValue("outfile","f","Output table",None)], - 'opt': [ParValue("energy","s","Monochromatic energy to use in PSF lookup [keV]",'broad'),ParRange("ecf","r","Encircled counts fraction (psf fraction)",0.9,0,1.0),ParValue("psffile","f","REEF Caldb filename",'CALDB'),ParRange("verbose","i","Tool chatter level",0,0,5),ParValue("clobber","b","Removing existing files?",False)], + 'opt': [ParValue("energy","s","Monochromatic energy to use in PSF lookup [keV]",'broad'),ParRange("ecf","r","Encircled counts fraction (psf fraction)",0.9,0,1),ParValue("psffile","f","REEF Caldb filename",'CALDB'),ParRange("verbose","i","Tool chatter level",0,0,5),ParValue("clobber","b","Removing existing files?",False)], } @@ -3741,7 +3741,7 @@ def make_tool(toolname): parinfo['tgdetect'] = { 'istool': True, 'req': [ParValue("infile","f","Input L1 event file",None),ParValue("OBI_srclist_file","f","Input source position(s) file from previous OBI or NONE",'NONE'),ParValue("outfile","f","Output source position(s) file name",None)], - 'opt': [ParValue("temproot","s","Path and root file name to be given to temporary files",None),ParValue("keeptemp","b","Keep temporary files?",False),ParValue("keepexit","b","Keep exit status file?",False),ParValue("zo_pos_x","s","Center GZO filter sky X position (default=pixel(ra_nom))",'default'),ParValue("zo_pos_y","s","Center GZO filter sky Y position (default=pixel(dec_nom))",'default'),ParValue("zo_sz_filt_x","s","Size of GZO filter in X pixels (ACIS=400; HRC=1800)",'default'),ParValue("zo_sz_filt_y","s","Size of GZO filter in Y pixels (ACIS=400; HRC=1800)",'default'),ParRange("snr_thresh","r","SNR threshold to select the detected sources",40,3,10000),ParValue("spectrum","f","Spectrum file [keV vs weight] if energy=None",None),ParValue("psffile","f","PSF Calibration file",'CALDB'),ParSet("units","s","Units of output image",'arcsec',["arcsec","logical","physical"]),ParValue("geompar","f","Pixlib geometry file",'geom.par'),ParValue("expstk","f","list of exposure map files",'none'),ParRange("thresh","r","celldetect source threshold",3,3,10000),ParValue("ellsigma","r","Size of output source ellipses (in sigmas)",3.0),ParValue("expratio","r","cutoff ratio for source cell exposure variation",0),ParValue("findpeaks","b","find local peaks for celldetect",True),ParRange("eband","r","energy band, for celldetect",1.4967,0,None),ParRange("eenergy","r","encircled energy of PSF, for celldetect",0.8,0,None),ParValue("celldetect_log","b","make a celldetect log file?",False),ParRange("fixedcell","i","celldetect fixed cell size to use",15,0,999),ParRange("fixedcell_cc_mode","i","celldetect fixed cell size to use for CC mode ACIS data",15,0,999),ParValue("snr_diminution","r","Diminution on SNR threshold - range (< 0 to 1) - Allows fine grained cell sliding",1.0),ParValue("bkgfile","f","background file, for celldetect",'none'),ParRange("bkgvalue","r","background count/pixel, for celldetect",0,0,None),ParRange("bkgerrvalue","r","background error, for celldetect",0,0,None),ParValue("snrfile","f","celldetect snr output file (for convolution only)",'none'),ParValue("convolve","b","use convolutions for celldetect",False),ParRange("xoffset","i","celldetect offset of x axis from optical axis",None,"INDEF","INDEF"),ParRange("yoffset","i","celldetect offset of y axis from optical axis",None,"INDEF","INDEF"),ParValue("cellfile","f","output cell size image file",'none'),ParValue("centroid","b","compute source centroids in celldetection?",True),ParRange("snr_ratio_limit","r","Value of SNR ratio to use as lower limit, for tgidselectsrc",1.0,0.0,1.0),ParValue("setsrcid","b","Set src ids in output file?, for tgidselectsrc",True),ParRange("max_separation","r","Maximum allowed separation (arcsec) for sources to match, for tgmatchsrc",3,0,20),ParValue("clobber","b","OK to overwrite existing output file(s)?",False),ParRange("verbose","i","Verbosity level (0 = no display)",0,0,5)], + 'opt': [ParValue("temproot","s","Path and root file name to be given to temporary files",None),ParValue("keeptemp","b","Keep temporary files?",False),ParValue("keepexit","b","Keep exit status file?",False),ParValue("zo_pos_x","s","Center GZO filter sky X position (default=pixel(ra_nom))",'default'),ParValue("zo_pos_y","s","Center GZO filter sky Y position (default=pixel(dec_nom))",'default'),ParValue("zo_sz_filt_x","s","Size of GZO filter in X pixels (ACIS=400; HRC=1800)",'default'),ParValue("zo_sz_filt_y","s","Size of GZO filter in Y pixels (ACIS=400; HRC=1800)",'default'),ParRange("snr_thresh","r","SNR threshold to select the detected sources",40,3,10000),ParValue("spectrum","f","Spectrum file [keV vs weight] if energy=None",None),ParValue("psffile","f","PSF Calibration file",'CALDB'),ParSet("units","s","Units of output image",'arcsec',["arcsec","logical","physical"]),ParValue("geompar","f","Pixlib geometry file",'geom.par'),ParValue("expstk","f","list of exposure map files",'none'),ParRange("thresh","r","celldetect source threshold",3,3,10000),ParValue("ellsigma","r","Size of output source ellipses (in sigmas)",3.0),ParValue("expratio","r","cutoff ratio for source cell exposure variation",0),ParValue("findpeaks","b","find local peaks for celldetect",True),ParRange("eband","r","energy band, for celldetect",1.4967,0,None),ParRange("eenergy","r","encircled energy of PSF, for celldetect",0.8,0,None),ParValue("celldetect_log","b","make a celldetect log file?",False),ParRange("fixedcell","i","celldetect fixed cell size to use",15,0,999),ParRange("fixedcell_cc_mode","i","celldetect fixed cell size to use for CC mode ACIS data",15,0,999),ParValue("snr_diminution","r","Diminution on SNR threshold - range (< 0 to 1) - Allows fine grained cell sliding",1.0),ParValue("bkgfile","f","background file, for celldetect",'none'),ParRange("bkgvalue","r","background count/pixel, for celldetect",0,0,None),ParRange("bkgerrvalue","r","background error, for celldetect",0,0,None),ParValue("snrfile","f","celldetect snr output file (for convolution only)",'none'),ParValue("convolve","b","use convolutions for celldetect",False),ParRange("xoffset","i","celldetect offset of x axis from optical axis",None,"INDEF","INDEF"),ParRange("yoffset","i","celldetect offset of y axis from optical axis",None,"INDEF","INDEF"),ParValue("cellfile","f","output cell size image file",'none'),ParValue("centroid","b","compute source centroids in celldetection?",True),ParRange("snr_ratio_limit","r","Value of SNR ratio to use as lower limit, for tgidselectsrc",1.0,0,1),ParValue("setsrcid","b","Set src ids in output file?, for tgidselectsrc",True),ParRange("max_separation","r","Maximum allowed separation (arcsec) for sources to match, for tgmatchsrc",3,0,20),ParValue("clobber","b","OK to overwrite existing output file(s)?",False),ParRange("verbose","i","Verbosity level (0 = no display)",0,0,5)], } @@ -3768,7 +3768,7 @@ def make_tool(toolname): parinfo['tgidselectsrc'] = { 'istool': True, - 'req': [ParValue("infile","f","Name of input src file",None),ParValue("outfile","f","Name of output filtered src file",None),ParRange("snr_ratio_limit","r","Value of SNR ratio to use as lower limit",1.0,0.0,1.0)], + 'req': [ParValue("infile","f","Name of input src file",None),ParValue("outfile","f","Name of output filtered src file",None),ParRange("snr_ratio_limit","r","Value of SNR ratio to use as lower limit",1.0,0,1)], 'opt': [ParValue("setsrcid","b","Set src ids?",True),ParValue("clobber","b","Clobber output file?",False),ParRange("verbose","i","Verbose level",0,0,5)], }