Add duration in result (allankp#32)

* Manage existing test run or test plan

* Fill the test failure comment and duration with information from test runner

* Log only last error content

* Add log

* Add log

* Fix: In case of 'parametrize', the test status reflects the last test performed (should reflect all the tests status)
Now, worst results are published at the end (sort by status)

* Manage existing testrun testplan (allankp#3)

Synchronization with upstream

* Merge branch 'fill-comment-in-test-result'

* Synchro with upstream

* Merge upstream

* Fix problem with Tox/Python versions

* Fix problem with Python 2.7

Author: apallier

pytest_testrail/plugin.py

@@ -1,5 +1,7 @@
 # -*- coding: UTF-8 -*-
 from datetime import datetime
+from operator import itemgetter
+
 import pytest
 import re
 import warnings
@@ -20,6 +22,8 @@
 GET_TESTRUN_URL = 'get_run/{}'
 GET_TESTPLAN_URL = 'get_plan/{}'
 
+COMMENT_SIZE_LIMIT = 4000
+
 
 class DeprecatedTestDecorator(DeprecationWarning):
     pass
@@ -150,44 +154,52 @@ def pytest_runtest_makereport(self, item, call):
         """ Collect result and associated testcases (TestRail) of an execution """
         outcome = yield
         rep = outcome.get_result()
-        comment = ""
-        if call.excinfo:
-            comment = str(item.repr_failure(call.excinfo))
         if item.get_marker(TESTRAIL_PREFIX):
             testcaseids = item.get_marker(TESTRAIL_PREFIX).kwargs.get('ids')
 
             if rep.when == 'call' and testcaseids:
                 self.add_result(
                     clean_test_ids(testcaseids),
                     get_test_outcome(outcome.result.outcome),
-                    comment)
+                    comment=rep.longrepr,
+                    duration=rep.duration
+                )
 
     def pytest_sessionfinish(self, session, exitstatus):
         """ Publish results in TestRail """
+        print('[{}] Start publishing'.format(TESTRAIL_PREFIX))
         if self.results:
+            tests_list = [str(result['case_id']) for result in self.results]
+            print('[{}] Testcases to publish: {}'.format(TESTRAIL_PREFIX, ', '.join(tests_list)))
 
             if self.testrun_id:
                 self.add_results(self.testrun_id)
             elif self.testplan_id:
-                for testrun_id in self.get_available_testruns(self.testplan_id):
+                testruns = self.get_available_testruns(self.testplan_id)
+                print('[{}] Testruns to update: {}'.format(TESTRAIL_PREFIX, ', '.join([str(elt) for elt in testruns])))
+                for testrun_id in testruns:
                     self.add_results(testrun_id)
             else:
-                print('No data published')
+                print('[{}] No data published'.format(TESTRAIL_PREFIX))
+        print('[{}] End publishing'.format(TESTRAIL_PREFIX))
 
     # plugin
 
-    def add_result(self, test_ids, status, comment):
+    def add_result(self, test_ids, status, comment='', duration=0):
         """
         Add a new result to results dict to be submitted at the end.
 
         :param list test_ids: list of test_ids.
         :param int status: status code of test (pass or fail).
+        :param comment: None or a failure representation.
+        :param duration: Time it took to run just the test.
         """
         for test_id in test_ids:
             data = {
                 'case_id': test_id,
-                'comment': comment,
                 'status_id': status,
+                'comment': comment,
+                'duration': duration
             }
             self.results.append(data)
 
@@ -198,18 +210,35 @@ def add_results(self, testrun_id):
         :param testrun_id: Id of the testrun to feed
 
         """
+        # Results are sorted by 'case_id' and by 'status_id' (worst result at the end)
+        self.results.sort(key=itemgetter('status_id'))
+        self.results.sort(key=itemgetter('case_id'))
+
+        # Publish results
         for result in self.results:
             data = {'status_id': result['status_id']}
             if self.version:
                 data['version'] = self.version
+            comment = result.get('comment', '')
+            if comment:
+                # Indent text to avoid string formatting by TestRail. Limit size of comment.
+                data['comment'] = "# Pytest result: #\n"
+                data['comment'] += 'Log truncated\n...\n' if len(str(comment)) > COMMENT_SIZE_LIMIT else ''
+                data['comment'] += "    " + str(comment)[-COMMENT_SIZE_LIMIT:].replace('\n', '\n    ')
+            duration = result.get('duration')
+            if duration:
+                duration = 1 if (duration < 1) else int(round(duration))  # TestRail API doesn't manage milliseconds
+                data['elapsed'] = str(duration) + 's'
             response = self.client.send_post(
                 ADD_RESULT_URL.format(testrun_id, result['case_id']),
                 data,
                 cert_check=self.cert_check
             )
             error = self.client.get_error(response)
             if error:
-                print('Info: Testcase #{} not published for following reason: "{}"'.format(result['case_id'], error))
+                print('[{}] Info: Testcase #{} not published for following reason: "{}"'.format(TESTRAIL_PREFIX,
+                                                                                                result['case_id'],
+                                                                                                error))
 
     def create_test_run(
             self, assign_user_id, project_id, suite_id, testrun_name, tr_keys):
@@ -233,10 +262,12 @@ def create_test_run(
         )
         error = self.client.get_error(response)
         if error:
-            print('Failed to create testrun: "{}"'.format(error))
+            print('[{}] Failed to create testrun: "{}"'.format(TESTRAIL_PREFIX, error))
         else:
             self.testrun_id = response['id']
-            print('New testrun created with name "{}" and ID={}'.format(testrun_name, self.testrun_id))
+            print('[{}] New testrun created with name "{}" and ID={}'.format(TESTRAIL_PREFIX,
+                                                                              testrun_name,
+                                                                              self.testrun_id))
 
     def is_testrun_available(self):
         """
@@ -250,10 +281,10 @@ def is_testrun_available(self):
         )
         error = self.client.get_error(response)
         if error:
-            print('Failed to retrieve testrun: "{}"'.format(error))
+            print('[{}] Failed to retrieve testrun: "{}"'.format(TESTRAIL_PREFIX, error))
             return False
-        else:
-            return response['is_completed'] is False
+
+        return response['is_completed'] is False
 
     def is_testplan_available(self):
         """
@@ -267,10 +298,10 @@ def is_testplan_available(self):
         )
         error = self.client.get_error(response)
         if error:
-            print('Failed to retrieve testplan: "{}"'.format(error))
+            print('[{}] Failed to retrieve testplan: "{}"'.format(TESTRAIL_PREFIX, error))
             return False
-        else:
-            return response['is_completed'] is False
+
+        return response['is_completed'] is False
 
     def get_available_testruns(self, plan_id):
         """
@@ -284,7 +315,7 @@ def get_available_testruns(self, plan_id):
         )
         error = self.client.get_error(response)
         if error:
-            print('Failed to retrieve testplan: "{}"'.format(error))
+            print('[{}] Failed to retrieve testplan: "{}"'.format(TESTRAIL_PREFIX, error))
         else:
             for entry in response['entries']:
                 for run in entry['runs']:

tests/livetest/livetest.py

@@ -1,24 +1,26 @@
 # -*- coding: UTF-8 -*-
 import pytest
+import time
 
 from pytest_testrail.plugin import testrail, pytestrail
 
 @testrail('C344', 'C366')
 def test_func1():
-    pass
+    time.sleep(0.5)
 
 @testrail('C345')
 def test_func2():
+    time.sleep(1.6)
     pytest.fail()
 
 @testrail('C99999')
 def test_func3():
-    pass
+    time.sleep(0.5)
 
 @pytestrail.case('C1788')
 def test_func4():
     pytest.skip()
 
 @pytestrail.case('C1789')
 def test_func5():
-    pass
+    time.sleep(0.5)