Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

updates to delete_sources & swarp #66

Merged
merged 8 commits into from
Jan 12, 2024
Merged

updates to delete_sources & swarp #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4aa9d17
ENH remove nullwt links in delete sources step
sidneymau Jan 11, 2024
1370819
ENH remove coadd object map file
sidneymau Jan 11, 2024
f534446
ENH remove tmp swarp files when debug != False
sidneymau Jan 11, 2024
8199629
ENH remove swarp files in delete_sources
sidneymau Jan 12, 2024
fe51092
BUG match swarp coadd files with regex
sidneymau Jan 12, 2024
7203111
BUG check if dir exists before listing/walking
sidneymau Jan 12, 2024
ae78bfc
linting
sidneymau Jan 12, 2024
f6ced46
ENH delete lists
sidneymau Jan 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 160 additions & 14 deletions eastlake/steps/delete_sources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from __future__ import print_function, absolute_import
import os
import re

from ..step import Step
from ..utils import safe_rm, safe_rmdir
from ..utils import safe_rm, safe_rmdir, get_relpath


class DeleteSources(Step):
Expand Down Expand Up @@ -43,6 +44,7 @@ def execute(self, stash, new_params=None):

# Now the per-band coadds
for band in stash["bands"]:

coadd_file = stash.get_filepaths(
"coadd_file", tilename, band=band,
keyerror=False,
Expand Down Expand Up @@ -72,6 +74,79 @@ def execute(self, stash, new_params=None):
self.logger.debug("removing file %s" % bkg_rms_file)
safe_rm(bkg_rms_file)

# ...and coadd object map
coadd_object_map_file = stash.get_filepaths(
"coadd_object_map", tilename, band=band,
keyerror=False,
)
if (coadd_object_map_file is not None):
if os.path.isfile(coadd_object_map_file):
self.logger.debug("removing file %s" % coadd_object_map_file)
safe_rm(coadd_object_map_file)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as far as I know, this isn't needed for anything downstream (and is leftover from srcextractor/true detection processing...?)—assuming safe to delete

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should run one sim with all of these recent PRs to make sure.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This wouldn't affect the sim as this is the last step after everything else is done. I think it's just a question of if the BFD folks need this particular file (from what I recall, they only mentioned needing the meds/fitvd stuff)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah we're good to go then.

self.logger.error("deleting swarp files %s" % tilename)
for band in stash["bands"]:
# Clean up any single-band files leftover from swarp
orig_coadd_path = stash.get_input_pizza_cutter_yaml(tilename, band)["image_path"]
coadd_path_from_imsim_data = get_relpath(
orig_coadd_path, stash["imsim_data"])
output_coadd_path = os.path.join(
base_dir, coadd_path_from_imsim_data)
if output_coadd_path.endswith("fits.fz"):
output_coadd_path = output_coadd_path[:-3]
output_coadd_dir = os.path.dirname(output_coadd_path)

output_coadd_sci_file = os.path.join(
output_coadd_dir, "%s_%s_sci.fits" % (tilename, band))
output_coadd_weight_file = os.path.join(
output_coadd_dir, "%s_%s_wgt.fits" % (tilename, band))
output_coadd_mask_file = os.path.join(
output_coadd_dir, "%s_%s_msk.fits" % (tilename, band))

safe_rm(output_coadd_sci_file)
safe_rm(output_coadd_weight_file)
safe_rm(output_coadd_mask_file)

dummy_mask_coadd = os.path.join(
output_coadd_dir, "%s_%s_msk-tmp.fits" % (tilename, band))
safe_rm(dummy_mask_coadd)

safe_rm(output_coadd_path + ".fz")

im_file_list = os.path.join(output_coadd_dir, "im_file_list.dat")
wgt_file_list = os.path.join(output_coadd_dir, "wgt_file_list.dat")
wgt_me_file_list = os.path.join(output_coadd_dir, "wgt_me_file_list.dat")
msk_file_list = os.path.join(output_coadd_dir, "msk_file_list.dat")
safe_rm(im_file_list)
safe_rm(wgt_file_list)
safe_rm(wgt_me_file_list)
safe_rm(msk_file_list)

# Remove swarp coadd files
# Match the coadd bands with regular expressions as the swarp
# coadds may be made with a different subset of bands
coadd_dir = os.path.join(
base_dir, stash["desrun"], tilename, "coadd")
if os.path.isdir(coadd_dir):
bands = "".join(stash["bands"])

det_coadd_file_re = re.compile(f"{tilename}_coadd_det_[{bands}]+.fits")
coadd_file_re = re.compile(f"{tilename}_coadd_[{bands}]+.fits")
weight_file_re = re.compile(f"{tilename}_coadd_weight_[{bands}]+.fits")
mask_tmp_file_re = re.compile(f"{tilename}_coadd_[{bands}]+_tmp.fits")
mask_file_re = re.compile(f"{tilename}_coadd_[{bands}]+_msk.fits")

for coadd_file in os.listdir(coadd_dir):
if (
det_coadd_file_re.fullmatch(coadd_file)
or coadd_file_re.fullmatch(coadd_file)
or weight_file_re.fullmatch(coadd_file)
or mask_tmp_file_re.fullmatch(coadd_file)
or mask_file_re.fullmatch(coadd_file)
):
coadd_file_path = os.path.join(coadd_dir, coadd_file)
safe_rm(coadd_file_path)

self.logger.error("deleting se images for tile %s" % tilename)
for band in stash["bands"]:
img_files = stash.get_filepaths(
Expand Down Expand Up @@ -123,33 +198,104 @@ def execute(self, stash, new_params=None):
self.logger.debug("removing file %s" % t)
safe_rm(t)

self.logger.error("removing psf links for %s" % tilename)
self.logger.error("deleting %s-band nullwt links for %s" % (band, tilename))

psf_link = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
os.path.basename(pyml["psf_path"])
)
safe_rm(psf_link)
for sri in pyml["src_info"]:
nullwt_link = os.path.join(
base_dir, stash["desrun"], tilename, f"nullwt-{band}",
os.path.basename(sri["coadd_nwgint_path"])
)
safe_rm(nullwt_link)

nullwt_path = os.path.join(
base_dir, stash["desrun"], tilename, f"nullwt-{band}",
)
safe_rmdir(nullwt_path)

self.logger.error("deleting %s-band psfs links for %s" % (band, tilename))

for sri in pyml["src_info"]:
psf_link = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
os.path.basename(sri["psf_path"])
os.path.basename(pyml["psf_path"])
)
safe_rm(psf_link)

for sri in pyml["src_info"]:
psf_link = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
os.path.basename(sri["psf_path"])
)
safe_rm(psf_link)

self.logger.error("deleting %s-band psfmap for %s" % (band, tilename))
pmap_fn = os.path.join(
base_dir, stash["desrun"], tilename,
f"{tilename}_{band}_psfmap-{stash['desrun']}.dat"
)
safe_rm(pmap_fn)

self.logger.error("deleting %s-band lists for %s" % (band, tilename))
fileconf = os.path.join(
base_dir, stash["desrun"], tilename, f"lists-{band}",
f"{tilename}_{band}_fileconf-{stash['desrun']}.yaml"
)
safe_rm(fileconf)
for listdir in ["lists", f"lists-{band}"]:
bkg_flist = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_bkg-flist-{stash['desrun']}.dat"
)
finalcut = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_finalcut-flist-{stash['desrun']}.dat"
)
nullwt = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_nullwt-flist-{stash['desrun']}.dat"
)
nwgint = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_nwgint-flist-{stash['desrun']}.dat"
)
piff = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_piff-flist-{stash['desrun']}.dat"
)
psf = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_psf-flist-{stash['desrun']}.dat"
)
seg = os.path.join(
base_dir, stash["desrun"], tilename, listdir,
f"{tilename}_{band}_seg-flist-{stash['desrun']}.dat"
)
safe_rm(bkg_flist)
safe_rm(finalcut)
safe_rm(nullwt)
safe_rm(nwgint)
safe_rm(piff)
safe_rm(psf)
safe_rm(seg)

psf_path = os.path.join(
base_dir, stash["desrun"], tilename, "psfs",
)
safe_rmdir(psf_path)

pmap_fn = os.path.join(
base_dir, stash["desrun"], tilename,
f"{tilename}_all_psfmap.dat"
)
safe_rm(pmap_fn)

self.logger.error("deleting empty dirs")

tile_path = os.path.join(base_dir, stash["desrun"], tilename)
for root, dirs, files in os.walk(tile_path, topdown=False):
for name in dirs:
full_dir = os.path.join(root, name)
if len(os.listdir(full_dir)) == 0:
safe_rmdir(full_dir)
if os.path.isdir(tile_path):
for root, dirs, files in os.walk(tile_path, topdown=False):
for name in dirs:
full_dir = os.path.join(root, name)
if len(os.listdir(full_dir)) == 0:
safe_rmdir(full_dir)

return 0, stash
33 changes: 22 additions & 11 deletions eastlake/steps/swarp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,10 @@ def execute(self, stash, new_params=None):

self._det_coadd(tilename, stash, band_coadd_data)

for band, fns in band_coadd_data.items():
for fn in fns.values():
safe_rm(fn)
if self.config.get("debug", False) is False:
for band, fns in band_coadd_data.items():
for fn in fns.values():
safe_rm(fn)

self.logger.error(
"%s complete for tile %s" % (self.name, tilename))
Expand Down Expand Up @@ -231,9 +232,10 @@ def _single_band_coadd(self, tilename, band, stash):
output_coadd_mask_file = os.path.join(
output_coadd_dir, "%s_%s_msk.fits" % (tilename, band))

safe_rm(output_coadd_sci_file)
safe_rm(output_coadd_weight_file)
safe_rm(output_coadd_mask_file)
if self.config.get("debug", False) is False:
safe_rm(output_coadd_sci_file)
safe_rm(output_coadd_weight_file)
safe_rm(output_coadd_mask_file)

# make the output directory and then move here to run swarp
# this prevents the intermediate files being fucked up by
Expand Down Expand Up @@ -330,7 +332,8 @@ def _single_band_coadd(self, tilename, band, stash):
"running swarp for tile %s, band %s w/ mask:\n\t%s" % (
tilename, band, " ".join(mask_cmd)))
run_and_check(mask_cmd, "Mask SWarp", logger=self.logger)
safe_rm(dummy_mask_coadd)
if self.config.get("debug", False) is False:
safe_rm(dummy_mask_coadd)

# now run coadd_assemble
safe_rm(output_coadd_path)
Expand Down Expand Up @@ -377,6 +380,13 @@ def _single_band_coadd(self, tilename, band, stash):
logger=self.logger
)

# Remove tmp swarp convenience files
if self.config.get("debug", False) is False:
safe_rm(im_file_list)
safe_rm(wgt_file_list)
safe_rm(wgt_me_file_list)
safe_rm(msk_file_list)

with stash.update_output_pizza_cutter_yaml(tilename, band) as pyml:
pyml["image_path"] = output_coadd_path + ".fz"
pyml["image_ext"] = "sci"
Expand Down Expand Up @@ -565,9 +575,10 @@ def _det_coadd(self, tilename, stash, band_coadd_data):
run_and_check(asmb_cmd, "coadd_assemble", logger=self.logger)

# remove tmp files
safe_rm(mask_tmp_file)
safe_rm(coadd_file)
safe_rm(weight_file)
safe_rm(mask_file)
if self.config.get("debug", False) is False:
safe_rm(mask_tmp_file)
safe_rm(coadd_file)
safe_rm(weight_file)
safe_rm(mask_file)

stash.set_filepaths("det_coadd_file", det_coadd_file, tilename)
Loading