Add option to output all output to file within INLClient

Author: loganharbour

client/JobRunner.py

@@ -300,6 +300,13 @@ def enqueue_output(out, queue):
         t.daemon = True
         t.start();
 
+        log_job_output_file = None
+        if 'log_job_output' in self.client_info and self.client_info['log_job_output'] is True:
+            if 'log_dir' in self.client_info and self.client_info['log_dir']:
+                log_job_output = os.path.join(self.client_info['log_dir'], 'job_{}_{}'.format(self.job_data['job_id'], step_data['step_num']))
+                log_job_output_file = open(log_job_output, 'wb')
+                logger.info('Writing job output to {}'.format(log_job_output))
+
         while proc.poll() is None:
             if self.canceled or self.stopped:
                 logger.info("Killing job\n")
@@ -309,9 +316,14 @@ def enqueue_output(out, queue):
                 break
 
             output = self.get_output_from_queue(q)
-            if output and not over_max:
-                out.extend(output)
-                chunk_out.extend(output)
+            if output:
+                if not over_max:
+                    out.extend(output)
+                    chunk_out.extend(output)
+                if log_job_output_file is not None:
+                    for chunk in output:
+                        log_job_output_file.write(chunk.encode('utf-8'))
+                    log_job_output_file.flush()
 
             # Don't worry that "out" might contain multibyte characters, we
             # just want a rough size check
@@ -343,9 +355,16 @@ def enqueue_output(out, queue):
         t.join() # make sure the step has no more output
 
         # we might not have gotten everything
-        out.extend(self.get_output_from_queue(q, timeout=0))
+        remaining_output = self.get_output_from_queue(q, timeout=0)
+        out.extend(remaining_output)
         if not step_data['canceled'] or keep_output:
             step_data['output'] = ''.join(out)
+        if log_job_output_file is not None:
+            if remaining_output:
+                for chunk in remaining_output:
+                    log_job_output_file.write(remaining_output.encode('utf-8'))
+                log_job_output_file.flush()
+            log_job_output_file.close()
         step_data['complete'] = True
         step_data['time'] = int(time.time() - start_time) #would be float
         return step_data

client/inl_client.py

@@ -54,7 +54,10 @@ def commandline_client(args):
             action='store_true',
             dest='user_client_suffix',
             help='Adds the user to client name as a suffix, i.e, sets the name to <hostname>_<user>_<client number>')
-
+    parser.add_argument("--log-job-output",
+            action='store_true',
+            default=False,
+            help='If set, logs all job output to the logging directory with the file prefix "job_"')
 
     parsed = parser.parse_args(args)
     home = os.environ.get("CIVET_HOME", os.path.join(os.environ["HOME"], "civet"))
@@ -72,6 +75,7 @@ def commandline_client(args):
         "ssl_cert": "",
         "log_file": "",
         "log_dir": log_dir,
+        "log_job_output": parsed.log_job_output,
         "build_key": "",
         "single_shot": False,
         "poll": 30,

client/tests/test_JobRunner.py

@@ -340,3 +340,15 @@ def test_max_step_time(self):
                 self.assertIn("taking longer than the max", out["output"])
                 self.assertLess(out["time"], 10)
                 self.assertEqual(out["canceled"], True)
+
+    def test_log_job_output(self):
+        with JobRunner.temp_file() as script:
+            script.write(b'echo "foo"\nsleep 5\necho "bar"')
+            script.close()
+            with open(os.devnull, "wb") as devnull:
+                r = self.create_runner()
+                r.client_info['log_job_output'] = True
+                proc = r.create_process(script.name, {}, devnull)
+                r.read_process_output(proc, r.job_data["steps"][0], {'step_num': 0})
+                with open(os.path.join(r.client_info['log_dir'], 'job_1_0'), 'r') as f:
+                    self.assertEqual(f.readlines(), ['foo\n', 'bar\n'])