Adding XML output to Python unittests.

gr_unittest.py was updated to handle the path and file creation and run the XML runner

gr_xmlrunner.py is a public domain program written by Sebastian Rittau <srittau@jroger.in-berlin.de> (THANKS!) to extend Python's unittest module to handle XML JUnit format output. This was renamed from xmlrunner.py if any future work is done on it from our project as well as to keep the namespace clean and clear.

qa_skiphead.py is a first test program for the XML output. Creates an outputfile $HOME/.gnuradio/unittest/python/test_skiphead.xml.

3 files changed, 420 insertions, 4 deletions

diff --git a/gnuradio-core/src/python/gnuradio/gr/qa_skiphead.py b/gnuradio-core/src/python/gnuradio/gr/qa_skiphead.py
index 106e97314..de2d8fc95 100755
--- a/gnuradio-core/src/python/gnuradio/gr/qa_skiphead.py
+++ b/gnuradio-core/src/python/gnuradio/gr/qa_skiphead.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2007 Free Software Foundation, Inc.
+# Copyright 2007,2010 Free Software Foundation, Inc.
 # 
 # This file is part of GNU Radio
 # 
@@ -99,4 +99,4 @@ class test_skiphead (gr_unittest.TestCase):
     
 
 if __name__ == '__main__':
-    gr_unittest.main ()
+    gr_unittest.run(test_skiphead, "test_skiphead.xml")
diff --git a/gnuradio-core/src/python/gnuradio/gr_unittest.py b/gnuradio-core/src/python/gnuradio/gr_unittest.py
index a48343c6b..29a731943 100755
--- a/gnuradio-core/src/python/gnuradio/gr_unittest.py
+++ b/gnuradio-core/src/python/gnuradio/gr_unittest.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2004 Free Software Foundation, Inc.
+# Copyright 2004,2010 Free Software Foundation, Inc.
 # 
 # This file is part of GNU Radio
 # 
@@ -21,7 +21,8 @@
 # 
 
 import unittest
-import sys
+import gr_xmlrunner
+import sys, os, stat
 
 class TestCase(unittest.TestCase):
     """A subclass of unittest.TestCase that adds additional assertions
@@ -106,6 +107,36 @@ TextTestRunner = unittest.TextTestRunner
 TestProgram = unittest.TestProgram
 main = TestProgram
 
+def run(PUT, filename=None):
+    ''' 
+    Runs the unittest on a TestCase and produces an optional XML report
+    PUT:      the program under test and should be a gr_unittest.TestCase
+    filename: an optional filename to save the XML report of the tests
+              this will live in $HOME/.gnuradio/unittests/python
+    '''
+
+    # Run this is given a file name
+    if(filename is not None):
+        path = os.getenv("HOME") + "/.gnuradio/unittests/python"
+
+        # Test if path exists; if not, build it
+        try:
+            st = os.stat(path)
+        except OSError:
+            os.makedirs(path, 0750)
+
+        # Create an XML runner to filename
+        fout = file(path+"/"+filename, "w")
+        runner = gr_xmlrunner.XMLTestRunner(fout)
+
+        # Run the test; runner also creates XML output file
+        suite = TestLoader().loadTestsFromTestCase(PUT)
+        runner.run(suite)
+    else:
+        # If no filename is given, just run the test
+        main()
+
+
 ##############################################################################
 # Executing this module from the command line
 ##############################################################################
diff --git a/gnuradio-core/src/python/gnuradio/gr_xmlrunner.py b/gnuradio-core/src/python/gnuradio/gr_xmlrunner.py
new file mode 100644
index 000000000..ded77f5f3
--- /dev/null
+++ b/gnuradio-core/src/python/gnuradio/gr_xmlrunner.py
@@ -0,0 +1,385 @@
+"""
+XML Test Runner for PyUnit
+"""
+
+# Written by Sebastian Rittau <srittau@jroger.in-berlin.de> and placed in
+# the Public Domain. With contributions by Paolo Borelli and others.
+# Added to GNU Radio Oct. 3, 2010
+
+from __future__ import with_statement
+
+__version__ = "0.1"
+
+import os.path
+import re
+import sys
+import time
+import traceback
+import unittest
+from xml.sax.saxutils import escape
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+
+class _TestInfo(object):
+
+    """Information about a particular test.
+    
+    Used by _XMLTestResult.
+    
+    """
+
+    def __init__(self, test, time):
+        (self._class, self._method) = test.id().rsplit(".", 1)
+        self._time = time
+        self._error = None
+        self._failure = None
+
+    @staticmethod
+    def create_success(test, time):
+        """Create a _TestInfo instance for a successful test."""
+        return _TestInfo(test, time)
+
+    @staticmethod
+    def create_failure(test, time, failure):
+        """Create a _TestInfo instance for a failed test."""
+        info = _TestInfo(test, time)
+        info._failure = failure
+        return info
+
+    @staticmethod
+    def create_error(test, time, error):
+        """Create a _TestInfo instance for an erroneous test."""
+        info = _TestInfo(test, time)
+        info._error = error
+        return info
+
+    def print_report(self, stream):
+        """Print information about this test case in XML format to the
+        supplied stream.
+
+        """
+        stream.write('  <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \
+            {
+                "class": self._class,
+                "method": self._method,
+                "time": self._time,
+            })
+        if self._failure is not None:
+            self._print_error(stream, 'failure', self._failure)
+        if self._error is not None:
+            self._print_error(stream, 'error', self._error)
+        stream.write('</testcase>\n')
+
+    def _print_error(self, stream, tagname, error):
+        """Print information from a failure or error to the supplied stream."""
+        text = escape(str(error[1]))
+        stream.write('\n')
+        stream.write('    <%s type="%s">%s\n' \
+            % (tagname, _clsname(error[0]), text))
+        tb_stream = StringIO()
+        traceback.print_tb(error[2], None, tb_stream)
+        stream.write(escape(tb_stream.getvalue()))
+        stream.write('    </%s>\n' % tagname)
+        stream.write('  ')
+
+
+def _clsname(cls):
+    return cls.__module__ + "." + cls.__name__
+
+
+class _XMLTestResult(unittest.TestResult):
+
+    """A test result class that stores result as XML.
+
+    Used by XMLTestRunner.
+
+    """
+
+    def __init__(self, classname):
+        unittest.TestResult.__init__(self)
+        self._test_name = classname
+        self._start_time = None
+        self._tests = []
+        self._error = None
+        self._failure = None
+
+    def startTest(self, test):
+        unittest.TestResult.startTest(self, test)
+        self._error = None
+        self._failure = None
+        self._start_time = time.time()
+
+    def stopTest(self, test):
+        time_taken = time.time() - self._start_time
+        unittest.TestResult.stopTest(self, test)
+        if self._error:
+            info = _TestInfo.create_error(test, time_taken, self._error)
+        elif self._failure:
+            info = _TestInfo.create_failure(test, time_taken, self._failure)
+        else:
+            info = _TestInfo.create_success(test, time_taken)
+        self._tests.append(info)
+
+    def addError(self, test, err):
+        unittest.TestResult.addError(self, test, err)
+        self._error = err
+
+    def addFailure(self, test, err):
+        unittest.TestResult.addFailure(self, test, err)
+        self._failure = err
+
+    def print_report(self, stream, time_taken, out, err):
+        """Prints the XML report to the supplied stream.
+        
+        The time the tests took to perform as well as the captured standard
+        output and standard error streams must be passed in.a
+
+        """
+        stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \
+            { "e": len(self.errors), "f": len(self.failures) })
+        stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \
+            {
+                "n": self._test_name,
+                "t": self.testsRun,
+                "time": time_taken,
+            })
+        for info in self._tests:
+            info.print_report(stream)
+        stream.write('  <system-out><![CDATA[%s]]></system-out>\n' % out)
+        stream.write('  <system-err><![CDATA[%s]]></system-err>\n' % err)
+        stream.write('</testsuite>\n')
+
+
+class XMLTestRunner(object):
+
+    """A test runner that stores results in XML format compatible with JUnit.
+
+    XMLTestRunner(stream=None) -> XML test runner
+
+    The XML file is written to the supplied stream. If stream is None, the
+    results are stored in a file called TEST-<module>.<class>.xml in the
+    current working directory (if not overridden with the path property),
+    where <module> and <class> are the module and class name of the test class.
+
+    """
+
+    def __init__(self, stream=None):
+        self._stream = stream
+        self._path = "."
+
+    def run(self, test):
+        """Run the given test case or test suite."""
+        class_ = test.__class__
+        classname = class_.__module__ + "." + class_.__name__
+        if self._stream == None:
+            filename = "TEST-%s.xml" % classname
+            stream = file(os.path.join(self._path, filename), "w")
+            stream.write('<?xml version="1.0" encoding="utf-8"?>\n')
+        else:
+            stream = self._stream
+
+        result = _XMLTestResult(classname)
+        start_time = time.time()
+
+        with _fake_std_streams():
+            test(result)
+            try:
+                out_s = sys.stdout.getvalue()
+            except AttributeError:
+                out_s = ""
+            try:
+                err_s = sys.stderr.getvalue()
+            except AttributeError:
+                err_s = ""
+
+        time_taken = time.time() - start_time
+        result.print_report(stream, time_taken, out_s, err_s)
+        if self._stream is None:
+            stream.close()
+
+        return result
+
+    def _set_path(self, path):
+        self._path = path
+
+    path = property(lambda self: self._path, _set_path, None,
+            """The path where the XML files are stored.
+            
+            This property is ignored when the XML file is written to a file
+            stream.""")
+
+
+class _fake_std_streams(object):
+
+    def __enter__(self):
+        self._orig_stdout = sys.stdout
+        self._orig_stderr = sys.stderr
+        sys.stdout = StringIO()
+        sys.stderr = StringIO()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout = self._orig_stdout
+        sys.stderr = self._orig_stderr
+
+
+class XMLTestRunnerTest(unittest.TestCase):
+
+    def setUp(self):
+        self._stream = StringIO()
+
+    def _try_test_run(self, test_class, expected):
+
+        """Run the test suite against the supplied test class and compare the
+        XML result against the expected XML string. Fail if the expected
+        string doesn't match the actual string. All time attributes in the
+        expected string should have the value "0.000". All error and failure
+        messages are reduced to "Foobar".
+
+        """
+
+        runner = XMLTestRunner(self._stream)
+        runner.run(unittest.makeSuite(test_class))
+
+        got = self._stream.getvalue()
+        # Replace all time="X.YYY" attributes by time="0.000" to enable a
+        # simple string comparison.
+        got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got)
+        # Likewise, replace all failure and error messages by a simple "Foobar"
+        # string.
+        got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got)
+        got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got)
+        # And finally Python 3 compatibility.
+        got = got.replace('type="builtins.', 'type="exceptions.')
+
+        self.assertEqual(expected, got)
+
+    def test_no_tests(self):
+        """Regression test: Check whether a test run without any tests
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            pass
+        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000">
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
+""")
+
+    def test_success(self):
+        """Regression test: Check whether a test run with a successful test
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                pass
+        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
+  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
+""")
+
+    def test_failure(self):
+        """Regression test: Check whether a test run with a failing test
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                self.assert_(False)
+        self._try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000">
+  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
+    <failure type="exceptions.AssertionError">Foobar</failure>
+  </testcase>
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
+""")
+
+    def test_error(self):
+        """Regression test: Check whether a test run with a erroneous test
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                raise IndexError()
+        self._try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
+  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
+    <error type="exceptions.IndexError">Foobar</error>
+  </testcase>
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
+""")
+
+    def test_stdout_capture(self):
+        """Regression test: Check whether a test run with output to stdout
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                sys.stdout.write("Test\n")
+        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
+  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
+  <system-out><![CDATA[Test
+]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
+""")
+
+    def test_stderr_capture(self):
+        """Regression test: Check whether a test run with output to stderr
+        matches a previous run.
+        
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                sys.stderr.write("Test\n")
+        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
+  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[Test
+]]></system-err>
+</testsuite>
+""")
+
+    class NullStream(object):
+        """A file-like object that discards everything written to it."""
+        def write(self, buffer):
+            pass
+
+    def test_unittests_changing_stdout(self):
+        """Check whether the XMLTestRunner recovers gracefully from unit tests
+        that change stdout, but don't change it back properly.
+
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                sys.stdout = XMLTestRunnerTest.NullStream()
+
+        runner = XMLTestRunner(self._stream)
+        runner.run(unittest.makeSuite(TestTest))
+
+    def test_unittests_changing_stderr(self):
+        """Check whether the XMLTestRunner recovers gracefully from unit tests
+        that change stderr, but don't change it back properly.
+
+        """
+        class TestTest(unittest.TestCase):
+            def test_foo(self):
+                sys.stderr = XMLTestRunnerTest.NullStream()
+
+        runner = XMLTestRunner(self._stream)
+        runner.run(unittest.makeSuite(TestTest))
+
+
+if __name__ == "__main__":
+    unittest.main()