test_result.py

"""
Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
See https://llvm.org/LICENSE.txt for license information.
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

Provides the LLDBTestResult class, which holds information about progress
and results of a single test run.
"""

# System modules
importos
importtraceback

# Third-party modules
importunittest

# LLDB Modules
from . importconfiguration
fromlldbsuite.test_eventimportbuild_exception


classLLDBTestResult(unittest.TextTestResult):
"""
 Enforce a singleton pattern to allow introspection of test progress.

 Overwrite addError(), addFailure(), and addExpectedFailure() methods
 to enable each test instance to track its failure/error status. It
 is used in the LLDB test framework to emit detailed trace messages
 to a log file for easier human inspection of test failures/errors.
 """

__singleton__=None
__ignore_singleton__=False

@staticmethod
defgetTerminalSize():
importos

env=os.environ

defioctl_GWINSZ(fd):
try:
importfcntl
importtermios
importstruct

cr=struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
except:
return
returncr

cr=ioctl_GWINSZ(0) orioctl_GWINSZ(1) orioctl_GWINSZ(2)
ifnotcr:
try:
fd=os.open(os.ctermid(), os.O_RDONLY)
cr=ioctl_GWINSZ(fd)
os.close(fd)
except:
pass
ifnotcr:
cr= (env.get("LINES", 25), env.get("COLUMNS", 80))
returnint(cr[1]), int(cr[0])

def__init__(self, *args):
ifnotLLDBTestResult.__ignore_singleton__andLLDBTestResult.__singleton__:
raiseException("LLDBTestResult instantiated more than once")
super(LLDBTestResult, self).__init__(*args)
LLDBTestResult.__singleton__=self
# Now put this singleton into the lldb module namespace.
configuration.test_result=self
# Computes the format string for displaying the counter.
counterWidth=len(str(configuration.suite.countTestCases()))
self.fmt="%"+str(counterWidth) +"d: "
self.indentation=" "* (counterWidth+2)
# This counts from 1 .. suite.countTestCases().
self.counter=0
 (width, height) =LLDBTestResult.getTerminalSize()

def_config_string(self, test):
compiler=getattr(test, "getCompiler", None)
arch=getattr(test, "getArchitecture", None)
return"%s-%s"% (compiler() ifcompilerelse"", arch() ifarchelse"")

def_exc_info_to_string(self, err, test):
"""Overrides superclass TestResult's method in order to append
 our test config info string to the exception info string."""
ifhasattr(test, "getArchitecture") andhasattr(test, "getCompiler"):
return"%sConfig=%s-%s"% (
super(LLDBTestResult, self)._exc_info_to_string(err, test),
test.getArchitecture(),
test.getCompiler(),
 )
else:
returnsuper(LLDBTestResult, self)._exc_info_to_string(err, test)

defgetDescription(self, test):
doc_first_line=test.shortDescription()
ifself.descriptionsanddoc_first_line:
return"\n".join((str(test), self.indentation+doc_first_line))
else:
returnstr(test)

def_getTestPath(self, test):
# Use test.test_filename if the test was created with
# lldbinline.MakeInlineTest().
iftestisNone:
return""
elifhasattr(test, "test_filename"):
returntest.test_filename
else:
importinspect

returninspect.getsourcefile(test.__class__)

def_getFileBasedCategories(self, test):
"""
 Returns the list of categories to which this test case belongs by
 collecting values of "categories" files. We start at the folder the test is in
 and traverse the hierarchy upwards until the test-suite root directory.
 """
start_path=self._getTestPath(test)

importos.path

folder=os.path.dirname(start_path)

fromlldbsuiteimportlldb_test_rootastest_root

iftest_root!=os.path.commonprefix([folder, test_root]):
raiseException(
"The test file %s is outside the test root directory"%start_path
 )

categories=set()
whilenotos.path.samefile(folder, test_root):
categories_file_name=os.path.join(folder, "categories")
ifos.path.exists(categories_file_name):
categories_file=open(categories_file_name, "r")
categories_str=categories_file.readline().strip()
categories_file.close()
categories.update(categories_str.split(","))
folder=os.path.dirname(folder)

returnlist(categories)

defgetCategoriesForTest(self, test):
"""
 Gets all the categories for the currently running test method in test case
 """
test_categories= []
test_categories.extend(getattr(test, "categories", []))

test_method=getattr(test, test._testMethodName)
iftest_methodisnotNone:
test_categories.extend(getattr(test_method, "categories", []))

test_categories.extend(self._getFileBasedCategories(test))

returntest_categories

defhardMarkAsSkipped(self, test):
getattr(test, test._testMethodName).__func__.__unittest_skip__=True
getattr(
test, test._testMethodName
 ).__func__.__unittest_skip_why__= (
"test case does not fall in any category of interest for this run"
 )

defcheckExclusion(self, exclusion_list, name):
ifexclusion_list:
importre

foriteminexclusion_list:
ifre.search(item, name):
returnTrue
returnFalse

defcheckCategoryExclusion(self, exclusion_list, test):
returnnotset(exclusion_list).isdisjoint(self.getCategoriesForTest(test))

defstartTest(self, test):
ifconfiguration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
self.hardMarkAsSkipped(test)
ifself.checkExclusion(configuration.skip_tests, test.id()):
self.hardMarkAsSkipped(test)

self.counter+=1
test.test_number=self.counter
ifself.showAll:
self.stream.write(self.fmt%self.counter)
super(LLDBTestResult, self).startTest(test)

defaddSuccess(self, test):
ifself.checkExclusion(
configuration.xfail_tests, test.id()
 ) orself.checkCategoryExclusion(configuration.xfail_categories, test):
self.addUnexpectedSuccess(test, None)
return

super(LLDBTestResult, self).addSuccess(test)
self.stream.write(
"PASS: LLDB (%s) :: %s\n"% (self._config_string(test), str(test))
 )

def_isBuildError(self, err_tuple):
exception=err_tuple[1]
returnisinstance(exception, build_exception.BuildError)

def_saveBuildErrorTuple(self, test, err):
# Adjust the error description so it prints the build command and build error
# rather than an uninformative Python backtrace.
build_error=err[1]
error_description="{}\nTest Directory:\n{}".format(
str(build_error), os.path.dirname(self._getTestPath(test))
 )
self.errors.append((test, error_description))
self._mirrorOutput=True

defaddError(self, test, err):
configuration.sdir_has_content=True
ifself._isBuildError(err):
self._saveBuildErrorTuple(test, err)
else:
super(LLDBTestResult, self).addError(test, err)

method=getattr(test, "markError", None)
ifmethod:
method()
self.stream.write(
"FAIL: LLDB (%s) :: %s\n"% (self._config_string(test), str(test))
 )

defaddCleanupError(self, test, err):
configuration.sdir_has_content=True
super(LLDBTestResult, self).addCleanupError(test, err)
method=getattr(test, "markCleanupError", None)
ifmethod:
method()
self.stream.write(
"CLEANUP ERROR: LLDB (%s) :: %s\n%s\n"
% (self._config_string(test), str(test), traceback.format_exc())
 )

defaddFailure(self, test, err):
ifself.checkExclusion(
configuration.xfail_tests, test.id()
 ) orself.checkCategoryExclusion(configuration.xfail_categories, test):
self.addExpectedFailure(test, err)
return

configuration.sdir_has_content=True
super(LLDBTestResult, self).addFailure(test, err)
method=getattr(test, "markFailure", None)
ifmethod:
method()
self.stream.write(
"FAIL: LLDB (%s) :: %s\n"% (self._config_string(test), str(test))
 )
ifconfiguration.use_categories:
test_categories=self.getCategoriesForTest(test)
forcategoryintest_categories:
ifcategoryinconfiguration.failures_per_category:
configuration.failures_per_category[category] = (
configuration.failures_per_category[category] +1
 )
else:
configuration.failures_per_category[category] =1

defaddExpectedFailure(self, test, err):
configuration.sdir_has_content=True
super(LLDBTestResult, self).addExpectedFailure(test, err)
method=getattr(test, "markExpectedFailure", None)
ifmethod:
method(err)
self.stream.write(
"XFAIL: LLDB (%s) :: %s\n"% (self._config_string(test), str(test))
 )

defaddSkip(self, test, reason):
configuration.sdir_has_content=True
super(LLDBTestResult, self).addSkip(test, reason)
method=getattr(test, "markSkippedTest", None)
ifmethod:
method()
self.stream.write(
"UNSUPPORTED: LLDB (%s) :: %s (%s) \n"
% (self._config_string(test), str(test), reason)
 )

defaddUnexpectedSuccess(self, test):
configuration.sdir_has_content=True
super(LLDBTestResult, self).addUnexpectedSuccess(test)
method=getattr(test, "markUnexpectedSuccess", None)
ifmethod:
method()
self.stream.write(
"XPASS: LLDB (%s) :: %s\n"% (self._config_string(test), str(test))
 )