Skip to content

Commit

Permalink
Merge branch 'tickets/DM-44128'
Browse files Browse the repository at this point in the history
  • Loading branch information
srp3rd committed Apr 30, 2024
2 parents b4bc196 + cab44e3 commit b9480be
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 2 deletions.
2 changes: 1 addition & 1 deletion python/lsst/ctrl/oods/fileIngester.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def __init__(self, config, csc=None):
scanInterval = self.config["scanInterval"]
seconds = TimeInterval.calculateTotalSeconds(scanInterval)

self.fileQueue = FileQueue(self.image_staging_dir, seconds)
self.fileQueue = FileQueue(self.image_staging_dir, seconds, csc)

butlerConfigs = self.config["butlers"]
if len(butlerConfigs) == 0:
Expand Down
9 changes: 8 additions & 1 deletion python/lsst/ctrl/oods/fileQueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ class FileQueue(object):
The number of seconds to wait between directory scans. Defaults to 1.
"""

def __init__(self, dir_path, scanInterval=1):
def __init__(self, dir_path, scanInterval=1, csc=None):
self.dir_path = dir_path
self.scanInterval = scanInterval
self.csc = csc

self.fileSet = set()
self.condition = asyncio.Condition()
Expand All @@ -57,13 +58,19 @@ async def queue_files(self):
loop = asyncio.get_running_loop()
# now, add all the currently known files to the queue
while True:
if self.csc:
self.csc.log.debug("Scanning for new files to ingest")
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
file_list = await loop.run_in_executor(pool, scanner.getAllFiles)
if self.csc:
self.csc.log.debug("done scanning for new files")

if file_list:
async with self.condition:
self.fileSet.update(file_list)
self.condition.notify_all()
if self.csc:
self.csc.log.debug("waiting %d seconds", self.scanInterval)
await asyncio.sleep(self.scanInterval)

async def dequeue_files(self):
Expand Down

0 comments on commit b9480be

Please sign in to comment.