aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Tests/xmlrunner.py
blob: d6838aa22ec80ab0b86117c32f7a7ae3ab5a33af (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
# -*- coding: utf-8 -*-

"""unittest-xml-reporting is a PyUnit-based TestRunner that can export test
results to XML files that can be consumed by a wide range of tools, such as
build systems, IDEs and Continuous Integration servers.

This module provides the XMLTestRunner class, which is heavily based on the
default TextTestRunner. This makes the XMLTestRunner very simple to use.

The script below, adapted from the unittest documentation, shows how to use
XMLTestRunner in a very simple way. In fact, the only difference between this
script and the original one is the last line:

import random
import unittest
import xmlrunner

class TestSequenceFunctions(unittest.TestCase):
    def setUp(self):
        self.seq = range(10)

    def test_shuffle(self):
        # make sure the shuffled sequence does not lose any elements
        random.shuffle(self.seq)
        self.seq.sort()
        self.assertEqual(self.seq, range(10))

    def test_choice(self):
        element = random.choice(self.seq)
        self.assertTrue(element in self.seq)

    def test_sample(self):
        self.assertRaises(ValueError, random.sample, self.seq, 20)
        for element in random.sample(self.seq, 5):
            self.assertTrue(element in self.seq)

if __name__ == '__main__':
    unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
"""

from __future__ import absolute_import

import os
import sys
import time
from unittest import TestResult, TextTestResult, TextTestRunner
import xml.dom.minidom
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO  # doesn't accept 'str' in Py2


class XMLDocument(xml.dom.minidom.Document):
    def createCDATAOrText(self, data):
        if ']]>' in data:
            return self.createTextNode(data)
        return self.createCDATASection(data)


class _TestInfo(object):
    """This class is used to keep useful information about the execution of a
    test method.
    """

    # Possible test outcomes
    (SUCCESS, FAILURE, ERROR) = range(3)

    def __init__(self, test_result, test_method, outcome=SUCCESS, err=None):
        "Create a new instance of _TestInfo."
        self.test_result = test_result
        self.test_method = test_method
        self.outcome = outcome
        self.err = err
        self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or ''
        self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or ''

    def get_elapsed_time(self):
        """Return the time that shows how long the test method took to
        execute.
        """
        return self.test_result.stop_time - self.test_result.start_time

    def get_description(self):
        "Return a text representation of the test method."
        return self.test_result.getDescription(self.test_method)

    def get_error_info(self):
        """Return a text representation of an exception thrown by a test
        method.
        """
        if not self.err:
            return ''
        return self.test_result._exc_info_to_string(
            self.err, self.test_method)


class _XMLTestResult(TextTestResult):
    """A test result class that can express test results in a XML report.

    Used by XMLTestRunner.
    """
    def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1,
                 elapsed_times=True):
        "Create a new instance of _XMLTestResult."
        TextTestResult.__init__(self, stream, descriptions, verbosity)
        self.successes = []
        self.callback = None
        self.elapsed_times = elapsed_times
        self.output_patched = False

    def _prepare_callback(self, test_info, target_list, verbose_str,
        short_str):
        """Append a _TestInfo to the given target list and sets a callback
        method to be called by stopTest method.
        """
        target_list.append(test_info)
        def callback():
            """This callback prints the test method outcome to the stream,
            as well as the elapsed time.
            """

            # Ignore the elapsed times for a more reliable unit testing
            if not self.elapsed_times:
                self.start_time = self.stop_time = 0

            if self.showAll:
                self.stream.writeln('(%.3fs) %s' % \
                    (test_info.get_elapsed_time(), verbose_str))
            elif self.dots:
                self.stream.write(short_str)
        self.callback = callback

    def _patch_standard_output(self):
        """Replace the stdout and stderr streams with string-based streams
        in order to capture the tests' output.
        """
        if not self.output_patched:
            (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr)
            self.output_patched = True
        (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \
            (StringIO(), StringIO())

    def _restore_standard_output(self):
        "Restore the stdout and stderr streams."
        (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr)
        self.output_patched = False

    def startTest(self, test):
        "Called before execute each test method."
        self._patch_standard_output()
        self.start_time = time.time()
        TestResult.startTest(self, test)

        if self.showAll:
            self.stream.write('  ' + self.getDescription(test))
            self.stream.write(" ... ")

    def stopTest(self, test):
        "Called after execute each test method."
        self._restore_standard_output()
        TextTestResult.stopTest(self, test)
        self.stop_time = time.time()

        if self.callback and callable(self.callback):
            self.callback()
            self.callback = None

    def addSuccess(self, test):
        "Called when a test executes successfully."
        self._prepare_callback(_TestInfo(self, test),
                               self.successes, 'OK', '.')

    def addFailure(self, test, err):
        "Called when a test method fails."
        self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err),
                               self.failures, 'FAIL', 'F')

    def addError(self, test, err):
        "Called when a test method raises an error."
        self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err),
                               self.errors, 'ERROR', 'E')

    def printErrorList(self, flavour, errors):
        "Write some information about the FAIL or ERROR to the stream."
        for test_info in errors:
            if isinstance(test_info, tuple):
                test_info, exc_info = test_info

            try:
                t = test_info.get_elapsed_time()
            except AttributeError:
                t = 0
            try:
                descr = test_info.get_description()
            except AttributeError:
                try:
                    descr = test_info.getDescription()
                except AttributeError:
                    descr = str(test_info)
            try:
                err_info = test_info.get_error_info()
            except AttributeError:
                err_info = str(test_info)

            self.stream.writeln(self.separator1)
            self.stream.writeln('%s [%.3fs]: %s' % (flavour, t, descr))
            self.stream.writeln(self.separator2)
            self.stream.writeln('%s' % err_info)

    def _get_info_by_testcase(self):
        """This method organizes test results by TestCase module. This
        information is used during the report generation, where a XML report
        will be generated for each TestCase.
        """
        tests_by_testcase = {}

        for tests in (self.successes, self.failures, self.errors):
            for test_info in tests:
                if not isinstance(test_info, _TestInfo):
                    print("Unexpected test result type: %r" % (test_info,))
                    continue
                testcase = type(test_info.test_method)

                # Ignore module name if it is '__main__'
                module = testcase.__module__ + '.'
                if module == '__main__.':
                    module = ''
                testcase_name = module + testcase.__name__

                if testcase_name not in tests_by_testcase:
                    tests_by_testcase[testcase_name] = []
                tests_by_testcase[testcase_name].append(test_info)

        return tests_by_testcase

    def _report_testsuite(suite_name, tests, xml_document):
        "Appends the testsuite section to the XML document."
        testsuite = xml_document.createElement('testsuite')
        xml_document.appendChild(testsuite)

        testsuite.setAttribute('name', str(suite_name))
        testsuite.setAttribute('tests', str(len(tests)))

        testsuite.setAttribute('time', '%.3f' %
            sum([e.get_elapsed_time() for e in tests]))

        failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE])
        testsuite.setAttribute('failures', str(failures))

        errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR])
        testsuite.setAttribute('errors', str(errors))

        return testsuite

    _report_testsuite = staticmethod(_report_testsuite)

    def _report_testcase(suite_name, test_result, xml_testsuite, xml_document):
        "Appends a testcase section to the XML document."
        testcase = xml_document.createElement('testcase')
        xml_testsuite.appendChild(testcase)

        testcase.setAttribute('classname', str(suite_name))
        testcase.setAttribute('name', test_result.test_method.shortDescription()
                              or getattr(test_result.test_method, '_testMethodName',
                                         str(test_result.test_method)))
        testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time())

        if (test_result.outcome != _TestInfo.SUCCESS):
            elem_name = ('failure', 'error')[test_result.outcome-1]
            failure = xml_document.createElement(elem_name)
            testcase.appendChild(failure)

            failure.setAttribute('type', str(test_result.err[0].__name__))
            failure.setAttribute('message', str(test_result.err[1]))

            error_info = test_result.get_error_info()
            failureText = xml_document.createCDATAOrText(error_info)
            failure.appendChild(failureText)

    _report_testcase = staticmethod(_report_testcase)

    def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr):
        "Appends the system-out and system-err sections to the XML document."
        systemout = xml_document.createElement('system-out')
        xml_testsuite.appendChild(systemout)

        systemout_text = xml_document.createCDATAOrText(stdout)
        systemout.appendChild(systemout_text)

        systemerr = xml_document.createElement('system-err')
        xml_testsuite.appendChild(systemerr)

        systemerr_text = xml_document.createCDATAOrText(stderr)
        systemerr.appendChild(systemerr_text)

    _report_output = staticmethod(_report_output)

    def generate_reports(self, test_runner):
        "Generates the XML reports to a given XMLTestRunner object."
        all_results = self._get_info_by_testcase()

        if type(test_runner.output) == str and not \
            os.path.exists(test_runner.output):
            os.makedirs(test_runner.output)

        for suite, tests in all_results.items():
            doc = XMLDocument()

            # Build the XML file
            testsuite = _XMLTestResult._report_testsuite(suite, tests, doc)
            stdout, stderr = [], []
            for test in tests:
                _XMLTestResult._report_testcase(suite, test, testsuite, doc)
                if test.stdout:
                    stdout.extend(['*****************', test.get_description(), test.stdout])
                if test.stderr:
                    stderr.extend(['*****************', test.get_description(), test.stderr])
            _XMLTestResult._report_output(test_runner, testsuite, doc,
                                          '\n'.join(stdout), '\n'.join(stderr))
            xml_content = doc.toprettyxml(indent='\t')

            if type(test_runner.output) is str:
                report_file = open('%s%sTEST-%s.xml' % \
                    (test_runner.output, os.sep, suite), 'w')
                try:
                    report_file.write(xml_content)
                finally:
                    report_file.close()
            else:
                # Assume that test_runner.output is a stream
                test_runner.output.write(xml_content)


class XMLTestRunner(TextTestRunner):
    """A test runner class that outputs the results in JUnit like XML files.
    """
    def __init__(self, output='.', stream=None, descriptions=True, verbose=False, elapsed_times=True):
        "Create a new instance of XMLTestRunner."
        if stream is None:
            stream = sys.stderr
        verbosity = (1, 2)[verbose]
        TextTestRunner.__init__(self, stream, descriptions, verbosity)
        self.output = output
        self.elapsed_times = elapsed_times

    def _make_result(self):
        """Create the TestResult object which will be used to store
        information about the executed tests.
        """
        return _XMLTestResult(self.stream, self.descriptions, \
            self.verbosity, self.elapsed_times)

    def run(self, test):
        "Run the given test case or test suite."
        # Prepare the test execution
        result = self._make_result()

        # Print a nice header
        self.stream.writeln()
        self.stream.writeln('Running tests...')
        self.stream.writeln(result.separator2)

        # Execute tests
        start_time = time.time()
        test(result)
        stop_time = time.time()
        time_taken = stop_time - start_time

        # Generate reports
        self.stream.writeln()
        self.stream.writeln('Generating XML reports...')
        result.generate_reports(self)

        # Print results
        result.printErrors()
        self.stream.writeln(result.separator2)
        run = result.testsRun
        self.stream.writeln("Ran %d test%s in %.3fs" %
            (run, run != 1 and "s" or "", time_taken))
        self.stream.writeln()

        # Error traces
        if not result.wasSuccessful():
            self.stream.write("FAILED (")
            failed, errored = (len(result.failures), len(result.errors))
            if failed:
                self.stream.write("failures=%d" % failed)
            if errored:
                if failed:
                    self.stream.write(", ")
                self.stream.write("errors=%d" % errored)
            self.stream.writeln(")")
        else:
            self.stream.writeln("OK")

        return result