run.py: Introduce metadata files for runs

This commit adds a new file emitted by the matrix runs, intended to
provide metadata about the run for reporting tools that are executed
later, such as Argus. Currently, the metadata file always contains the
driver name (derived from result xml name), type (in this case it's
always python), and either failure_reason key (which contains exception
stack trace that happened during .run() or path to the resulting junit
xml file relative to the metadata file.

This commit is part of ongoing task to improve matrix pipeline reporting
to Argus.

Task: scylladb/argus#289

Author: k0machi

main.py

@@ -19,16 +19,16 @@ def main(arguments: argparse.Namespace):
     for driver_version in arguments.versions:
         for protocol in arguments.protocols:
             logging.info('=== PYTHON DRIVER VERSION %s, PROTOCOL v%s ===', driver_version, protocol)
-            try:
-                result = Run(python_driver_git=arguments.python_driver_git,
+            runner = Run(python_driver_git=arguments.python_driver_git,
                              python_driver_type=arguments.driver_type,
                              scylla_install_dir=arguments.scylla_install_dir,
                              tag=driver_version,
                              protocol=protocol,
                              tests=arguments.tests,
                              scylla_version=arguments.scylla_version,
-                             collect_only=arguments.collect_only).run()
-
+                             collect_only=arguments.collect_only)
+            try:
+                result = runner.run()
                 logging.info("=== (%s:%s) PYTHON DRIVER MATRIX RESULTS FOR PROTOCOL v%s ===",
                              arguments.driver_type, driver_version, protocol)
                 logging.info(", ".join(f"{key}: {value}" for key, value in result.summary.items()))
@@ -43,7 +43,9 @@ def main(arguments: argparse.Namespace):
                 logging.exception(f"{driver_version} failed")
                 status = 1
                 exc_type, exc_value, exc_traceback = sys.exc_info()
-                results[(driver_version, protocol)] = dict(exception=traceback.format_exception(exc_type, exc_value, exc_traceback))
+                failure_reason = traceback.format_exception(exc_type, exc_value, exc_traceback)
+                results[(driver_version, protocol)] = dict(exception=failure_reason)
+                runner.create_metadata_for_failure(reason="\n".join(failure_reason))
 
     if arguments.recipients:
         email_report = create_report(results=results)

run.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import re
@@ -52,13 +53,25 @@ def version_folder(self) -> Path:
         else:
             raise ValueError("Not found directory for python-driver version '%s'", self.driver_version)
 
+    @cached_property
+    def xunit_dir(self) -> Path:
+        return Path(os.path.dirname(__file__)) / "xunit" / self.driver_version
+
+    @property
+    def result_file_name(self) -> str:
+        return f'pytest.{self._python_driver_type}.v{self._protocol}.{self.driver_version}.xml'
+
+    @property
+    def metadata_file_name(self) -> str:
+        return f'metadata_{self._python_driver_type}_v{self._protocol}_{self.driver_version}.json'
+
     @cached_property
     def xunit_file(self) -> Path:
-        xunit_dir = Path(os.path.dirname(__file__)) / "xunit" / self.driver_version
-        if not xunit_dir.exists():
-            xunit_dir.mkdir(parents=True)
+        
+        if not self.xunit_dir.exists():
+            self.xunit_dir.mkdir(parents=True)
 
-        file_path = xunit_dir / f'pytest.{self._python_driver_type}.v{self._protocol}.{self.driver_version}.xml'
+        file_path = self.xunit_dir / self.result_file_name
         if file_path.exists():
             file_path.unlink()
         return file_path
@@ -166,8 +179,24 @@ def _checkout_branch(self):
             logging.error("Failed to branch for version '%s', with: '%s'", self.driver_version, str(exc))
             return False
 
+    def create_metadata_for_failure(self, reason: str) -> None:
+        metadata_file = self.xunit_dir / self.metadata_file_name
+        metadata = {
+            "driver_name": self.result_file_name.replace(".xml", ""),
+            "driver_type": "python",
+            "failure_reason": reason,
+        }
+        metadata_file.write_text(json.dumps(metadata))
+
+
     def run(self) -> ProcessJUnit:
         junit = ProcessJUnit(self.xunit_file, self.ignore_tests)
+        metadata_file = self.xunit_dir / self.metadata_file_name
+        metadata = {
+            "driver_name": self.result_file_name.replace(".xml", ""),
+            "driver_type": "python",
+            "junit_result": f"./{self.xunit_file.name}",
+        }
         logging.info("Changing the current working directory to the '%s' path", self._python_driver_git)
         os.chdir(self._python_driver_git)
         if self._checkout_branch() and self._apply_patch_files() and self._install_python_requirements():
@@ -180,6 +209,7 @@ def run(self) -> ProcessJUnit:
                             env=self.environment, cwd=self._python_driver_git)
             # clean ccm clusters, for next runs
             self._run_command_in_shell("rm -rf tests/integration/ccm | true")
+            metadata_file.write_text(json.dumps(metadata))
             junit.save_after_analysis(driver_version=self.driver_version, protocol=self._protocol,
                                       python_driver_type=self._python_driver_type)
         return junit