path: root/contrib/tools/cython/Cython/Tests/xmlrunner.py

# -*- coding: utf-8 -*- 
 
"""unittest-xml-reporting is a PyUnit-based TestRunner that can export test 
results to XML files that can be consumed by a wide range of tools, such as 
build systems, IDEs and Continuous Integration servers. 
 
This module provides the XMLTestRunner class, which is heavily based on the 
default TextTestRunner. This makes the XMLTestRunner very simple to use. 
 
The script below, adapted from the unittest documentation, shows how to use 
XMLTestRunner in a very simple way. In fact, the only difference between this 
script and the original one is the last line: 
 
import random 
import unittest 
import xmlrunner 
 
class TestSequenceFunctions(unittest.TestCase): 
    def setUp(self): 
        self.seq = range(10) 
 
    def test_shuffle(self): 
        # make sure the shuffled sequence does not lose any elements 
        random.shuffle(self.seq) 
        self.seq.sort() 
        self.assertEqual(self.seq, range(10)) 
 
    def test_choice(self): 
        element = random.choice(self.seq) 
        self.assertTrue(element in self.seq)
 
    def test_sample(self): 
        self.assertRaises(ValueError, random.sample, self.seq, 20) 
        for element in random.sample(self.seq, 5): 
            self.assertTrue(element in self.seq)
 
if __name__ == '__main__': 
    unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) 
""" 
 
from __future__ import absolute_import

import os 
import sys 
import time 
from unittest import TestResult, TextTestResult, TextTestRunner
import xml.dom.minidom 
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO  # doesn't accept 'str' in Py2
 
 
class XMLDocument(xml.dom.minidom.Document): 
    def createCDATAOrText(self, data): 
        if ']]>' in data: 
            return self.createTextNode(data) 
        return self.createCDATASection(data) 
 
 
class _TestInfo(object): 
    """This class is used to keep useful information about the execution of a 
    test method. 
    """ 
 
    # Possible test outcomes 
    (SUCCESS, FAILURE, ERROR) = range(3) 
 
    def __init__(self, test_result, test_method, outcome=SUCCESS, err=None): 
        "Create a new instance of _TestInfo." 
        self.test_result = test_result 
        self.test_method = test_method 
        self.outcome = outcome 
        self.err = err 
        self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or '' 
        self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or '' 
 
    def get_elapsed_time(self): 
        """Return the time that shows how long the test method took to 
        execute. 
        """ 
        return self.test_result.stop_time - self.test_result.start_time 
 
    def get_description(self): 
        "Return a text representation of the test method." 
        return self.test_result.getDescription(self.test_method) 
 
    def get_error_info(self): 
        """Return a text representation of an exception thrown by a test 
        method. 
        """ 
        if not self.err: 
            return '' 
        return self.test_result._exc_info_to_string( 
            self.err, self.test_method) 
 
 
class _XMLTestResult(TextTestResult):
    """A test result class that can express test results in a XML report. 
 
    Used by XMLTestRunner. 
    """ 
    def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, 
                 elapsed_times=True): 
        "Create a new instance of _XMLTestResult." 
        TextTestResult.__init__(self, stream, descriptions, verbosity)
        self.successes = [] 
        self.callback = None 
        self.elapsed_times = elapsed_times 
        self.output_patched = False 
 
    def _prepare_callback(self, test_info, target_list, verbose_str, 
        short_str): 
        """Append a _TestInfo to the given target list and sets a callback 
        method to be called by stopTest method. 
        """ 
        target_list.append(test_info) 
        def callback(): 
            """This callback prints the test method outcome to the stream, 
            as well as the elapsed time. 
            """ 
 
            # Ignore the elapsed times for a more reliable unit testing 
            if not self.elapsed_times: 
                self.start_time = self.stop_time = 0 
 
            if self.showAll: 
                self.stream.writeln('(%.3fs) %s' % \ 
                    (test_info.get_elapsed_time(), verbose_str)) 
            elif self.dots: 
                self.stream.write(short_str) 
        self.callback = callback 
 
    def _patch_standard_output(self): 
        """Replace the stdout and stderr streams with string-based streams 
        in order to capture the tests' output. 
        """ 
        if not self.output_patched: 
            (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr) 
            self.output_patched = True 
        (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \ 
            (StringIO(), StringIO()) 
 
    def _restore_standard_output(self): 
        "Restore the stdout and stderr streams." 
        (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr) 
        self.output_patched = False 
 
    def startTest(self, test): 
        "Called before execute each test method." 
        self._patch_standard_output() 
        self.start_time = time.time() 
        TestResult.startTest(self, test) 
 
        if self.showAll: 
            self.stream.write('  ' + self.getDescription(test)) 
            self.stream.write(" ... ") 
 
    def stopTest(self, test): 
        "Called after execute each test method." 
        self._restore_standard_output() 
        TextTestResult.stopTest(self, test)
        self.stop_time = time.time() 
 
        if self.callback and callable(self.callback): 
            self.callback() 
            self.callback = None 
 
    def addSuccess(self, test): 
        "Called when a test executes successfully." 
        self._prepare_callback(_TestInfo(self, test), 
                               self.successes, 'OK', '.') 
 
    def addFailure(self, test, err): 
        "Called when a test method fails." 
        self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err), 
                               self.failures, 'FAIL', 'F') 
 
    def addError(self, test, err): 
        "Called when a test method raises an error." 
        self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err), 
                               self.errors, 'ERROR', 'E') 
 
    def printErrorList(self, flavour, errors): 
        "Write some information about the FAIL or ERROR to the stream." 
        for test_info in errors: 
            if isinstance(test_info, tuple): 
                test_info, exc_info = test_info 

            try:
                t = test_info.get_elapsed_time()
            except AttributeError:
                t = 0
            try:
                descr = test_info.get_description()
            except AttributeError:
                try:
                    descr = test_info.getDescription()
                except AttributeError:
                    descr = str(test_info)
            try:
                err_info = test_info.get_error_info()
            except AttributeError:
                err_info = str(test_info)

            self.stream.writeln(self.separator1) 
            self.stream.writeln('%s [%.3fs]: %s' % (flavour, t, descr))
            self.stream.writeln(self.separator2) 
            self.stream.writeln('%s' % err_info)
 
    def _get_info_by_testcase(self): 
        """This method organizes test results by TestCase module. This 
        information is used during the report generation, where a XML report 
        will be generated for each TestCase. 
        """ 
        tests_by_testcase = {} 
 
        for tests in (self.successes, self.failures, self.errors): 
            for test_info in tests: 
                if not isinstance(test_info, _TestInfo):
                    print("Unexpected test result type: %r" % (test_info,))
                    continue
                testcase = type(test_info.test_method) 
 
                # Ignore module name if it is '__main__' 
                module = testcase.__module__ + '.' 
                if module == '__main__.': 
                    module = '' 
                testcase_name = module + testcase.__name__ 
 
                if testcase_name not in tests_by_testcase: 
                    tests_by_testcase[testcase_name] = [] 
                tests_by_testcase[testcase_name].append(test_info) 
 
        return tests_by_testcase 
 
    def _report_testsuite(suite_name, tests, xml_document): 
        "Appends the testsuite section to the XML document." 
        testsuite = xml_document.createElement('testsuite') 
        xml_document.appendChild(testsuite) 
 
        testsuite.setAttribute('name', str(suite_name)) 
        testsuite.setAttribute('tests', str(len(tests))) 
 
        testsuite.setAttribute('time', '%.3f' % 
            sum([e.get_elapsed_time() for e in tests])) 
 
        failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE]) 
        testsuite.setAttribute('failures', str(failures)) 
 
        errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR]) 
        testsuite.setAttribute('errors', str(errors)) 
 
        return testsuite 
 
    _report_testsuite = staticmethod(_report_testsuite) 
 
    def _report_testcase(suite_name, test_result, xml_testsuite, xml_document): 
        "Appends a testcase section to the XML document." 
        testcase = xml_document.createElement('testcase') 
        xml_testsuite.appendChild(testcase) 
 
        testcase.setAttribute('classname', str(suite_name)) 
        testcase.setAttribute('name', test_result.test_method.shortDescription() 
                              or getattr(test_result.test_method, '_testMethodName', 
                                         str(test_result.test_method))) 
        testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time()) 
 
        if (test_result.outcome != _TestInfo.SUCCESS): 
            elem_name = ('failure', 'error')[test_result.outcome-1] 
            failure = xml_document.createElement(elem_name) 
            testcase.appendChild(failure) 
 
            failure.setAttribute('type', str(test_result.err[0].__name__)) 
            failure.setAttribute('message', str(test_result.err[1])) 
 
            error_info = test_result.get_error_info() 
            failureText = xml_document.createCDATAOrText(error_info) 
            failure.appendChild(failureText) 
 
    _report_testcase = staticmethod(_report_testcase) 
 
    def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr): 
        "Appends the system-out and system-err sections to the XML document." 
        systemout = xml_document.createElement('system-out') 
        xml_testsuite.appendChild(systemout) 
 
        systemout_text = xml_document.createCDATAOrText(stdout) 
        systemout.appendChild(systemout_text) 
 
        systemerr = xml_document.createElement('system-err') 
        xml_testsuite.appendChild(systemerr) 
 
        systemerr_text = xml_document.createCDATAOrText(stderr) 
        systemerr.appendChild(systemerr_text) 
 
    _report_output = staticmethod(_report_output) 
 
    def generate_reports(self, test_runner): 
        "Generates the XML reports to a given XMLTestRunner object." 
        all_results = self._get_info_by_testcase() 
 
        if type(test_runner.output) == str and not \ 
            os.path.exists(test_runner.output): 
            os.makedirs(test_runner.output) 
 
        for suite, tests in all_results.items(): 
            doc = XMLDocument() 
 
            # Build the XML file 
            testsuite = _XMLTestResult._report_testsuite(suite, tests, doc) 
            stdout, stderr = [], [] 
            for test in tests: 
                _XMLTestResult._report_testcase(suite, test, testsuite, doc) 
                if test.stdout: 
                    stdout.extend(['*****************', test.get_description(), test.stdout]) 
                if test.stderr: 
                    stderr.extend(['*****************', test.get_description(), test.stderr]) 
            _XMLTestResult._report_output(test_runner, testsuite, doc, 
                                          '\n'.join(stdout), '\n'.join(stderr)) 
            xml_content = doc.toprettyxml(indent='\t') 
 
            if type(test_runner.output) is str: 
                report_file = open('%s%sTEST-%s.xml' % \ 
                    (test_runner.output, os.sep, suite), 'w') 
                try: 
                    report_file.write(xml_content) 
                finally: 
                    report_file.close() 
            else: 
                # Assume that test_runner.output is a stream 
                test_runner.output.write(xml_content) 
 
 
class XMLTestRunner(TextTestRunner): 
    """A test runner class that outputs the results in JUnit like XML files. 
    """ 
    def __init__(self, output='.', stream=None, descriptions=True, verbose=False, elapsed_times=True):
        "Create a new instance of XMLTestRunner." 
        if stream is None:
            stream = sys.stderr
        verbosity = (1, 2)[verbose] 
        TextTestRunner.__init__(self, stream, descriptions, verbosity) 
        self.output = output 
        self.elapsed_times = elapsed_times 
 
    def _make_result(self): 
        """Create the TestResult object which will be used to store 
        information about the executed tests. 
        """ 
        return _XMLTestResult(self.stream, self.descriptions, \ 
            self.verbosity, self.elapsed_times) 
 
    def run(self, test): 
        "Run the given test case or test suite." 
        # Prepare the test execution 
        result = self._make_result() 
 
        # Print a nice header 
        self.stream.writeln() 
        self.stream.writeln('Running tests...') 
        self.stream.writeln(result.separator2) 
 
        # Execute tests 
        start_time = time.time() 
        test(result) 
        stop_time = time.time() 
        time_taken = stop_time - start_time 
 
        # Generate reports
        self.stream.writeln()
        self.stream.writeln('Generating XML reports...')
        result.generate_reports(self)

        # Print results 
        result.printErrors() 
        self.stream.writeln(result.separator2) 
        run = result.testsRun 
        self.stream.writeln("Ran %d test%s in %.3fs" % 
            (run, run != 1 and "s" or "", time_taken)) 
        self.stream.writeln() 
 
        # Error traces 
        if not result.wasSuccessful(): 
            self.stream.write("FAILED (") 
            failed, errored = (len(result.failures), len(result.errors)) 
            if failed: 
                self.stream.write("failures=%d" % failed) 
            if errored: 
                if failed: 
                    self.stream.write(", ") 
                self.stream.write("errors=%d" % errored) 
            self.stream.writeln(")") 
        else: 
            self.stream.writeln("OK") 
 
        return result