Skip to content

Commit

Permalink
Implement parsing of test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@multiplemonomials
multiplemonomials committed Apr 20, 2024
1 parent 7be5183 commit 1ff3997
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 36 deletions.
0 ...st-configs/mbed-tests-NUCLEO_U575ZI-Q.xml → ...st-configs/mbed-tests-NUCLEO_U575ZI_Q.xml
View file
File renamed without changes.
68 changes: 38 additions & 30 deletions Test-Result-Evaluator/test_result_evaluator/mbed_test_database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,36 @@ def create_database(self):

self._database.execute("BEGIN")

# -- Targets table
# Lists targets and their attibutes
self._database.execute(
"CREATE TABLE Targets("
"name TEXT PRIMARY KEY, " # Name of the target
"isPublic INTEGER, " # 1 if the target is a public target (a board that can be built for).
# 0 if not (the target is just used as a parent for other targets).
"isMCUFamily INTEGER, " # 1 if this is the top-level target for one family of MCUs and boards.
# (for example, STM32L4 is the family target for all STM32L4 MCUs and boards).
# Family targets are used to group together documentation and test results.
# 0 otherwise (target is above or below the family level).
"cpuVendorName TEXT, " # Name of the vendor which the CPU on this target comes from. From the CMSIS
# database. NULL if this target does not have a valid link to the CMSIS database.
"mcuFamilyTarget TEXT NULL, " # Name of the MCU family target which is a parent of this target.
# Only set iff a target is or has a parent which is an MCU family target.
# If isMCUFamily = 1, this will contain the target's own name.
"imageURL TEXT NULL " # URL of the image that will be shown in the target table for this board, if set
# in JSON.
")"
)

# -- Tests table
# Holds details about tests for each target
self._database.execute(
"CREATE TABLE Tests("
"testName TEXT, " # Name of the test
"targetName TEXT, " # Name of the target it was ran for
"targetName TEXT REFERENCES Targets(name), " # Name of the target it was ran for
"executionTime REAL, " # Time in seconds it took to run the test
"result INTEGER," # TestResult of the test
"output TEXT," # Output that the complete test printed (not divided into test cases)
"FOREIGN KEY(targetName) REFERENCES Targets(name), "
"UNIQUE(testName, targetName)" # Combo of test name - target name must be unique
")"
)
Expand All @@ -79,12 +99,12 @@ def create_database(self):
# Holds the result of each test case for each target
self._database.execute(
"CREATE TABLE TestCases("
"testName TEXT, " # Name of the test
"testCaseName TEXT, " # Name of the test case
"targetName TEXT, " # Name of the target it was ran for
"result INTEGER," # TestResult of the test
"output TEXT," # Output that this test case printed specifically
"FOREIGN KEY(targetName) REFERENCES Targets(name), "
"testName TEXT NOT NULL, " # Name of the test
"testCaseName TEXT NOT NULL, " # Name of the test case
"targetName TEXT NOT NULL REFERENCES Targets(name), " # Name of the target it was ran for
"result INTEGER NOT NULL," # TestResult of the test
"output TEXT NOT NULL," # Output that this test case printed specifically, or empty string for skipped tests
"FOREIGN KEY(testName, targetName) REFERENCES Tests(testName, targetName), "
"UNIQUE(testName, testCaseName, targetName)" # Combo of test name - test case name - target name must be unique
")"
)
Expand All @@ -102,27 +122,6 @@ def create_database(self):
")"
)

# -- Targets table
# Lists targets and their attibutes
self._database.execute(
"CREATE TABLE Targets("
"name TEXT PRIMARY KEY, " # Name of the target
"isPublic INTEGER, " # 1 if the target is a public target (a board that can be built for).
# 0 if not (the target is just used as a parent for other targets).
"isMCUFamily INTEGER, " # 1 if this is the top-level target for one family of MCUs and boards.
# (for example, STM32L4 is the family target for all STM32L4 MCUs and boards).
# Family targets are used to group together documentation and test results.
# 0 otherwise (target is above or below the family level).
"cpuVendorName TEXT, " # Name of the vendor which the CPU on this target comes from. From the CMSIS
# database. NULL if this target does not have a valid link to the CMSIS database.
"mcuFamilyTarget TEXT NULL, " # Name of the MCU family target which is a parent of this target.
# Only set iff a target is or has a parent which is an MCU family target.
# If isMCUFamily = 1, this will contain the target's own name.
"imageURL TEXT NULL " # URL of the image that will be shown in the target table for this board, if set
# in JSON.
")"
)

# -- TargetGraph table
# Contains target parent/child relationships
self._database.execute(
Expand Down Expand Up @@ -577,4 +576,13 @@ def add_test_record(self, test_name: str, target_name: str, execution_time: floa
"""
self._database.execute("INSERT OR REPLACE INTO Tests(testName, targetName, executionTime, result, output) "
"VALUES(?, ?, ?, ?, ?)",
(test_name, target_name, execution_time, result.value, output))
(test_name, target_name, execution_time, result.value, output))

def add_test_case_record(self, test_name: str, test_case_name: str, target_name: str, result: TestResult, output: str):
"""
Add a record of a test to the TestCases table.
Replaces the record if it already exists
"""
self._database.execute("INSERT OR REPLACE INTO TestCases(testName, testCaseName, targetName, result, output) "
"VALUES(?, ?, ?, ?, ?)",
(test_name, test_case_name, target_name, result.value, output))
41 changes: 35 additions & 6 deletions Test-Result-Evaluator/test_result_evaluator/parse_test_run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,21 @@
if test_suite_result != TestResult.SKIPPED:
# Now things get a bit more complicated as we have to parse Greentea's output directly to determine
# the list of tests.

# First use a regex to extract the list of test cases...
test_case_names = re.findall(GREENTEA_TESTCASE_NAME_RE, test_report.system_out)

# Next, we need some special handling for tests which reset. These tests print out the list of
# test cases multiple times, which causes the previous operation to return duplicate results. Remove those
# while preserving the test case order.
test_case_names = list(dict.fromkeys(test_case_names))

test_case_records: List[Tuple[str, str]]

if len(test_case_names) > 0:
# This is a "normal" test with test cases. Parse them.
# Regex returns tuple of (output, passed/failed indicator)
test_case_records: List[Tuple[str, str]] = re.findall(GREENTEA_TESTCASE_OUTPUT_RE, test_report.system_out)
test_case_records = re.findall(GREENTEA_TESTCASE_OUTPUT_RE, test_report.system_out)

if len(test_case_records) < len(test_case_names):
# Did one test case crash the test?
Expand All @@ -66,16 +74,37 @@
crash_re = re.compile(r"\{\{__testcase_start;" + crashing_test_name + r"}}(.+?)teardown\(\) finished", re.DOTALL)
test_case_crash_output = re.search(crash_re, test_report.system_out)

if test_case_crash_output is None:
raise RuntimeError(f"Parse error: found {len(test_case_names)} test cases but only found {len(test_case_records)} records of test cases executing")
else:

if test_case_crash_output is not None:
print(f"Note: Test case '{crashing_test_name}' in test {test_report.classname} appears to have crashed and prevented {len(test_case_names) - len(test_case_records) - 1} subsequent tests from running")
test_case_records.append((test_case_crash_output.group(0), "0"))
test_case_records.append((test_case_crash_output.group(0), "0"))
else:
# Otherwise the test simply didn't run the remaining test cases.
pass

# However, there are some tests (e.g. test-mbed-drivers-dev-null) which don't use the greentea
# system in a standard way and therefore can't be divided evenly into test cases. These tests need special
# handling.
else:
print(f"This test has non-standard output. Treating the entire test as one test case")
test_case_records = [test_report.system_out, test_suite_result]

for test_case_idx, test_case_name in enumerate(test_case_names):

# If the test actually was run, save its output
if test_case_idx < len(test_case_records):
database.add_test_case_record(test_report.classname,
test_case_name,
mbed_target,
TestResult.PASSED if test_case_records[test_case_idx][1] == "1" else TestResult.FAILED,
test_case_records[test_case_idx][0])

# Otherwise, mark it as skipped
else:
database.add_test_case_record(test_report.classname,
test_case_name,
mbed_target,
TestResult.SKIPPED,
"")

print(">> Done.")
database.close()
Expand Down

0 comments on commit 1ff3997

Please sign in to comment.