Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test suite optimization: Migrate tests to pytest #917

Closed
wants to merge 25 commits into from
Closed

Test suite optimization: Migrate tests to pytest #917

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
96e7ca4
migrate test_cgroups.py
Naman-Priyadarshi Jun 30, 2023
26fec82
migrate test_benchmark_definition.py
Naman-Priyadarshi Jun 30, 2023
34d3580
update setup.cfg
Naman-Priyadarshi Jun 30, 2023
ee6519c
migrate test_analyze_run_result.py
Naman-Priyadarshi Jun 30, 2023
46d9b5e
migrate test_results.py
Naman-Priyadarshi Jul 10, 2023
2f981f1
forgot to add copyright text to test_result.py
Naman-Priyadarshi Jul 10, 2023
e75dff6
resolve changes
Naman-Priyadarshi Jul 25, 2023
d9e6a39
migrate test_util.py
Naman-Priyadarshi Jul 31, 2023
5149bad
add pytest to dockerfile
Sep 11, 2023
d9f5be9
reformat tests
Sep 11, 2023
601cb77
reformat tests
Sep 11, 2023
1b8e51a
replace nose with pytest in workflow
Sep 11, 2023
ab25f20
migrate test_statvalue.py and test_util.py
Naman-Priyadarshi Oct 17, 2023
891ea7c
reformat code
Naman-Priyadarshi Oct 17, 2023
44fc7d0
Merge branch 'migrate-test-to-pytest' of https://github.com/sosy-lab/…
Naman-Priyadarshi Oct 17, 2023
5f6fdfa
ruff changes
Naman-Priyadarshi Oct 17, 2023
a155144
Merge branch 'main' into migrate-test-to-pytest
Naman-Priyadarshi Nov 4, 2023
d218182
add file conftest.py
Nov 4, 2023
2359da2
use old util tests
Nov 4, 2023
7d1be54
reformat code
Nov 4, 2023
463105f
ruff changes
Nov 4, 2023
0041058
add license to test_statvalue.py
Nov 4, 2023
ca47383
make changes to appveyor workflow to support pytest
Nov 4, 2023
d73d101
update dockerfile
Nov 8, 2023
6d2b876
add comment explaining conftest.py
Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 72 additions & 70 deletions benchexec/test_analyze_run_result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import logging
import sys
import unittest
import pytest
import types

from benchexec.util import ProcessExitCode
Expand All @@ -24,13 +24,7 @@

normal_result = ProcessExitCode(raw=0, value=0, signal=None)


class TestResult(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
logging.disable(logging.CRITICAL)

class TestResult:
def create_run(self, info_result=RESULT_UNKNOWN):
runSet = types.SimpleNamespace()
runSet.log_folder = "."
Expand Down Expand Up @@ -64,158 +58,166 @@ def determine_result(run):

def test_simple(self):
run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual(RESULT_UNKNOWN, run._analyze_result(normal_result, "", None))
assert RESULT_UNKNOWN == run._analyze_result(normal_result, "", None)

run = self.create_run(info_result=RESULT_TRUE_PROP)
self.assertEqual(RESULT_TRUE_PROP, run._analyze_result(normal_result, "", None))
assert RESULT_TRUE_PROP == run._analyze_result(normal_result, "", None)

run = self.create_run(info_result=RESULT_FALSE_REACH)
self.assertEqual(
RESULT_FALSE_REACH, run._analyze_result(normal_result, "", None)
assert (
RESULT_FALSE_REACH
== run._analyze_result(normal_result, "", None)
)

def test_timeout(self):
run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", "cputime"))
self.assertEqual(
"TIMEOUT", run._analyze_result(normal_result, "", "cputime-soft")
assert "TIMEOUT" == run._analyze_result(normal_result, "", "cputime")
assert (
"TIMEOUT"
== run._analyze_result(normal_result, "", "cputime-soft")
)
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", "walltime"))
assert "TIMEOUT" == run._analyze_result(normal_result, "", "walltime")

run = self.create_run(info_result=RESULT_TRUE_PROP)
self.assertEqual(
f"TIMEOUT ({RESULT_TRUE_PROP})",
run._analyze_result(normal_result, "", "cputime"),
assert (
f"TIMEOUT ({RESULT_TRUE_PROP})"
== run._analyze_result(normal_result, "", "cputime")
)

run = self.create_run(info_result=RESULT_FALSE_REACH)
self.assertEqual(
f"TIMEOUT ({RESULT_FALSE_REACH})",
run._analyze_result(normal_result, "", "cputime"),
assert (
f"TIMEOUT ({RESULT_FALSE_REACH})"
== run._analyze_result(normal_result, "", "cputime")
)

run = self.create_run(info_result="SOME OTHER RESULT")
self.assertEqual(
"TIMEOUT (SOME OTHER RESULT)",
run._analyze_result(normal_result, "", "cputime"),
assert (
"TIMEOUT (SOME OTHER RESULT)"
== run._analyze_result(normal_result, "", "cputime")
)

run = self.create_run(info_result=RESULT_ERROR)
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", "cputime"))
assert "TIMEOUT" == run._analyze_result(normal_result, "", "cputime")

run = self.create_run(info_result=RESULT_ERROR)
run._is_timeout = lambda: True
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", None))
assert "TIMEOUT" == run._analyze_result(normal_result, "", None)

def test_out_of_memory(self):
run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual(
"OUT OF MEMORY", run._analyze_result(normal_result, "", "memory")
assert (
"OUT OF MEMORY"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result=RESULT_TRUE_PROP)
self.assertEqual(
f"OUT OF MEMORY ({RESULT_TRUE_PROP})",
run._analyze_result(normal_result, "", "memory"),
assert (
f"OUT OF MEMORY ({RESULT_TRUE_PROP})"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result=RESULT_FALSE_REACH)
self.assertEqual(
f"OUT OF MEMORY ({RESULT_FALSE_REACH})",
run._analyze_result(normal_result, "", "memory"),
assert (
f"OUT OF MEMORY ({RESULT_FALSE_REACH})"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result="SOME OTHER RESULT")
self.assertEqual(
"OUT OF MEMORY (SOME OTHER RESULT)",
run._analyze_result(normal_result, "", "memory"),
assert (
"OUT OF MEMORY (SOME OTHER RESULT)"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result=RESULT_ERROR)
self.assertEqual(
"OUT OF MEMORY", run._analyze_result(normal_result, "", "memory")
assert (
"OUT OF MEMORY"
== run._analyze_result(normal_result, "", "memory")
)

def test_timeout_and_out_of_memory(self):
run = self.create_run(info_result=RESULT_UNKNOWN)
run._is_timeout = lambda: True
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", "memory"))
assert "TIMEOUT" == run._analyze_result(normal_result, "", "memory")

run = self.create_run(info_result=RESULT_TRUE_PROP)
run._is_timeout = lambda: True
self.assertEqual(
f"TIMEOUT ({RESULT_TRUE_PROP})",
run._analyze_result(normal_result, "", "memory"),
assert (
f"TIMEOUT ({RESULT_TRUE_PROP})"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result=RESULT_FALSE_REACH)
run._is_timeout = lambda: True
self.assertEqual(
f"TIMEOUT ({RESULT_FALSE_REACH})",
run._analyze_result(normal_result, "", "memory"),
assert (
f"TIMEOUT ({RESULT_FALSE_REACH})"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result="SOME OTHER RESULT")
run._is_timeout = lambda: True
self.assertEqual(
"TIMEOUT (SOME OTHER RESULT)",
run._analyze_result(normal_result, "", "memory"),
assert (
"TIMEOUT (SOME OTHER RESULT)"
== run._analyze_result(normal_result, "", "memory")
)

run = self.create_run(info_result=RESULT_ERROR)
run._is_timeout = lambda: True
self.assertEqual("TIMEOUT", run._analyze_result(normal_result, "", "memory"))
assert "TIMEOUT" == run._analyze_result(normal_result, "", "memory")

def test_returnsignal(self):
def signal(sig):
"""Encode a signal as it would be returned by os.wait"""
return ProcessExitCode(raw=sig, value=None, signal=sig)

run = self.create_run(info_result=RESULT_ERROR)
self.assertEqual("TIMEOUT", run._analyze_result(signal(9), "", "cputime"))
assert "TIMEOUT" == run._analyze_result(signal(9), "", "cputime")

run = self.create_run(info_result=RESULT_ERROR)
self.assertEqual("OUT OF MEMORY", run._analyze_result(signal(9), "", "memory"))
assert (
"OUT OF MEMORY"
== run._analyze_result(signal(9), "", "memory")
)

run = self.create_run(info_result=RESULT_TRUE_PROP)
self.assertEqual(RESULT_TRUE_PROP, run._analyze_result(signal(9), "", None))
assert RESULT_TRUE_PROP == run._analyze_result(signal(9), "", None)

run = self.create_run(info_result=RESULT_FALSE_REACH)
self.assertEqual(RESULT_FALSE_REACH, run._analyze_result(signal(9), "", None))
assert RESULT_FALSE_REACH == run._analyze_result(signal(9), "", None)

run = self.create_run(info_result="SOME OTHER RESULT")
self.assertEqual("SOME OTHER RESULT", run._analyze_result(signal(9), "", None))
assert (
"SOME OTHER RESULT"
== run._analyze_result(signal(9), "", None)
)

run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual("KILLED BY SIGNAL 9", run._analyze_result(signal(9), "", None))
assert "KILLED BY SIGNAL 9" == run._analyze_result(signal(9), "", None)

def test_exitcode(self):
def returnvalue(value):
"""Encode an exit of aprogram as it would be returned by os.wait"""
"""Encode an exit of a program as it would be returned by os.wait"""
return ProcessExitCode(raw=value << 8, value=value, signal=None)

run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual("TIMEOUT", run._analyze_result(returnvalue(1), "", "cputime"))
assert "TIMEOUT" == run._analyze_result(returnvalue(1), "", "cputime")

run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual(
"OUT OF MEMORY", run._analyze_result(returnvalue(1), "", "memory")
assert (
"OUT OF MEMORY"
== run._analyze_result(returnvalue(1), "", "memory")
)

run = self.create_run(info_result=RESULT_TRUE_PROP)
self.assertEqual(
RESULT_TRUE_PROP, run._analyze_result(returnvalue(1), "", None)
)
assert RESULT_TRUE_PROP == run._analyze_result(returnvalue(1), "", None)

run = self.create_run(info_result=RESULT_FALSE_REACH)
self.assertEqual(
RESULT_FALSE_REACH, run._analyze_result(returnvalue(1), "", None)
)
assert RESULT_FALSE_REACH == run._analyze_result(returnvalue(1), "", None)

run = self.create_run(info_result="SOME OTHER RESULT")
self.assertEqual(
"SOME OTHER RESULT", run._analyze_result(returnvalue(1), "", None)
assert (
"SOME OTHER RESULT"
== run._analyze_result(returnvalue(1), "", None)
)

run = self.create_run(info_result=RESULT_UNKNOWN)
self.assertEqual(RESULT_UNKNOWN, run._analyze_result(returnvalue(1), "", None))
assert RESULT_UNKNOWN == run._analyze_result(returnvalue(1), "", None)
Loading