diff --git a/CHANGELOG.md b/CHANGELOG.md index 92a4c34..899df52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,13 @@ All notable changes to this project will be documented in this file. ### Changed * Don't install solvers by default from PyPI +* Move solver issue and timeout listings to test-set themselves * Rename the project "qpbenchmark" +### Removed + +* Module-wide skip solver issue/timeout functions + ## [1.2.0] - 2023-11-27 ### Added diff --git a/README.md b/README.md index 08c025d..c617a8c 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Contributions to improving this benchmark are welcome. You can for instance prop ## Citation -To cite this benchmark in your scientific works: +If you use `qpbenchmark` in your scientific works, please cite it *e.g.* as follows: ```bibtex @software{qpbenchmark_2023, @@ -148,7 +148,7 @@ To cite this benchmark in your scientific works: } ``` -You can also click on the ``Cite this repository`` button on GitHub, at the top-right of the [repository page](https://github.com/qpsolvers/qpbenchmark/). +You can also click on ``Cite this repository`` at the top-right of the [repository page](https://github.com/qpsolvers/qpbenchmark/). ## See also diff --git a/qpbenchmark/run.py b/qpbenchmark/run.py index 505a465..361b0e4 100644 --- a/qpbenchmark/run.py +++ b/qpbenchmark/run.py @@ -24,7 +24,6 @@ from qpsolvers.exceptions import SolverNotFound from .results import Results -from .solver_issues import skip_solver_issue, skip_solver_timeout from .spdlog import logging from .test_set import TestSet from .utils import time_solve_problem @@ -80,7 +79,7 @@ def run( for solver in filtered_solvers: for settings in filtered_settings: time_limit = test_set.tolerances[settings].runtime - if skip_solver_issue(problem, solver): + if (problem, solver) in test_set.known_solver_issues: failure = ( problem, solver, @@ -90,7 +89,9 @@ def run( ) results.update(*failure) continue - if skip_solver_timeout(time_limit, problem, solver, settings): + if test_set.skip_solver_timeout( + time_limit, problem, solver, settings + ): failure = ( problem, solver, diff --git a/qpbenchmark/solver_issues.py b/qpbenchmark/solver_issues.py index 41d2ead..746e75f 100644 --- a/qpbenchmark/solver_issues.py +++ b/qpbenchmark/solver_issues.py @@ -16,122 +16,3 @@ # limitations under the License. """Some solvers fail on some problems. Make sure we handle and report those.""" - -from typing import Dict, Tuple - -from .problem import Problem -from .spdlog import logging - - -def skip_solver_issue(problem: Problem, solver: str) -> bool: - """Check whether a solver is known to fail at solving a given problem. - - Args: - problem: Problem to solve. - solver: QP solver. - - Returns: - True if solver is known to fail at solving the problem. - """ - if solver == "proxqp" and problem.name == "QGFRDXPN": - # https://github.com/Simple-Robotics/proxsuite/issues/63 - return True - if solver == "highs" and problem.name == "STADAT1": - # https://github.com/ERGO-Code/HiGHS/issues/995 - return True - return False - - -def skip_solver_timeout( - time_limit: float, problem: Problem, solver: str, settings: str -) -> bool: - """Skip known solver timeouts. - - Args: - time_limit: Time limit in seconds. - problem: Problem to solve. - solver: QP solver. - settings: QP solver settings. - - Note: - This function only checks for timeouts that the solvers are not able to - handle by themselves, e.g. for those who do not provide a time limit - parameter. - - Returns: - True if `solver` is known to take more than `time_limit` seconds on - `problem` when using the solver parameters defined in `settings`. - """ - minutes = 60.0 - # TODO: these timeouts should be moved to test sets themselves - known_timeout_settings: Dict[Tuple[str, str, str], float] = { - ("AUG2D", "highs", "*"): 40 * minutes, - ("AUG2D", "proxqp", "high_accuracy"): 40 * minutes, - ("AUG2DC", "highs", "*"): 40 * minutes, - ("AUG2DC", "proxqp", "high_accuracy"): 40 * minutes, - ("AUG2DCQP", "proxqp", "high_accuracy"): 20 * minutes, - ("AUG2DQP", "proxqp", "high_accuracy"): 30 * minutes, - ("BOYD1", "proxqp", "*"): 30 * minutes, - ("BOYD2", "cvxopt", "mid_accuracy"): 30 * minutes, - ("BOYD2", "proxqp", "*"): 20 * minutes, - ("CONT-101", "proxqp", "*"): 30 * minutes, - ("CONT-200", "proxqp", "*"): 20 * minutes, - ("CONT-201", "proxqp", "*"): 30 * minutes, - ("CONT-300", "cvxopt", "*"): 20 * minutes, - ("CONT-300", "highs", "default"): 30 * minutes, - ("CONT-300", "highs", "high_accuracy"): 30 * minutes, - ("CONT-300", "proxqp", "*"): 60 * minutes, - ("CVXQP1_L", "proxqp", "*"): 20 * minutes, - ("CVXQP2_L", "proxqp", "high_accuracy"): 30 * minutes, - ("CVXQP3_L", "cvxopt", "*"): 20 * minutes, - ("CVXQP3_L", "proxqp", "*"): 30 * minutes, - ("DTOC3", "proxqp", "high_accuracy"): 20 * minutes, - ("DTOC3", "proxqp", "low_accuracy"): 20 * minutes, - ("DTOC3", "proxqp", "mid_accuracy"): 20 * minutes, - ("EXDATA", "proxqp", "*"): 30 * minutes, - ("LISWET1", "proxqp", "*"): 20 * minutes, - ("LISWET10", "proxqp", "*"): 50 * minutes, - ("LISWET11", "proxqp", "high_accuracy"): 40 * minutes, - ("LISWET11", "proxqp", "mid_accuracy"): 30 * minutes, - ("LISWET12", "proxqp", "high_accuracy"): 20 * minutes, - ("LISWET12", "proxqp", "low_accuracy"): 60 * minutes, - ("LISWET12", "proxqp", "mid_accuracy"): 60 * minutes, - ("LISWET2", "proxqp", "high_accuracy"): 20 * minutes, - ("LISWET2", "proxqp", "mid_accuracy"): 20 * minutes, - ("LISWET3", "proxqp", "high_accuracy"): 30 * minutes, - ("LISWET4", "proxqp", "high_accuracy"): 20 * minutes, - ("LISWET4", "proxqp", "mid_accuracy"): 30 * minutes, - ("LISWET5", "proxqp", "high_accuracy"): 20 * minutes, - ("LISWET6", "proxqp", "high_accuracy"): 20 * minutes, - ("LISWET6", "proxqp", "mid_accuracy"): 20 * minutes, - ("LISWET7", "proxqp", "high_accuracy"): 30 * minutes, - ("LISWET7", "proxqp", "mid_accuracy"): 30 * minutes, - ("LISWET8", "proxqp", "high_accuracy"): 30 * minutes, - ("LISWET8", "proxqp", "mid_accuracy"): 30 * minutes, - ("LISWET9", "proxqp", "high_accuracy"): 30 * minutes, - ("LISWET9", "proxqp", "low_accuracy"): 30 * minutes, - ("LISWET9", "proxqp", "mid_accuracy"): 30 * minutes, - ("POWELL20", "proxqp", "*"): 30 * minutes, - ("QGFRDXPN", "proxqp", "*"): 20 * minutes, - ("QSHIP08L", "proxqp", "*"): 20 * minutes, - ("QSHIP12L", "proxqp", "*"): 20 * minutes, - ("STADAT1", "proxqp", "*"): 20 * minutes, - ("STADAT2", "proxqp", "*"): 20 * minutes, - ("STADAT3", "proxqp", "*"): 20 * minutes, - ("UBH1", "proxqp", "*"): 20 * minutes, - ("YAO", "proxqp", "*"): 20 * minutes, - } - timeout = ( - known_timeout_settings[(problem.name, solver, settings)] - if (problem.name, solver, settings) in known_timeout_settings - else known_timeout_settings[(problem.name, solver, "*")] - if (problem.name, solver, "*") in known_timeout_settings - else 0.0 - ) - if timeout > time_limit: - logging.warning( - f"Skipping {problem.name} with {solver} at {settings} " - f"as it is known to take {timeout} > {time_limit} seconds..." - ) - return True - return False diff --git a/qpbenchmark/test_set.py b/qpbenchmark/test_set.py index e00691d..b0ef1b6 100644 --- a/qpbenchmark/test_set.py +++ b/qpbenchmark/test_set.py @@ -18,7 +18,7 @@ """Base class for test sets.""" import abc -from typing import Dict, Iterator, Optional +from typing import Dict, Iterator, Optional, Set, Tuple import qpsolvers @@ -39,6 +39,8 @@ class TestSet(abc.ABC): tolerances: Validation tolerances. """ + known_solver_issues: Set[Tuple[str, str]] + known_solver_timeouts: Dict[Tuple[str, str, str], float] solver_settings: Dict[str, SolverSettings] tolerances: Dict[str, Tolerance] @@ -136,6 +138,8 @@ def __init__(self): f"Solver '{solver}' is available but skipped " "as its settings are unknown" ) + self.known_solver_issues = set() + self.known_solver_timeouts = {} self.solver_settings = {} self.solvers = solvers self.tolerances = {} @@ -193,3 +197,38 @@ def get_problem(self, name: str) -> Optional[Problem]: f"problem '{name}' not found " f"in the {self.__class__.__name__} test set" ) + + def skip_solver_timeout( + self, time_limit: float, problem: Problem, solver: str, settings: str + ) -> bool: + """Skip known solver timeouts. + + Args: + time_limit: Time limit in seconds. + problem: Problem to solve. + solver: QP solver. + settings: QP solver settings. + + Note: + This function only checks for timeouts that the solvers are not + able to handle by themselves, e.g. for those who do not provide a + time limit parameter. + + Returns: + True if `solver` is known to take more than `time_limit` seconds on + `problem` when using the solver parameters defined in `settings`. + """ + timeout = ( + self.known_solver_timeouts[(problem.name, solver, settings)] + if (problem.name, solver, settings) in self.known_solver_timeouts + else self.known_solver_timeouts[(problem.name, solver, "*")] + if (problem.name, solver, "*") in self.known_solver_timeouts + else 0.0 + ) + if timeout > time_limit: + logging.warning( + f"Skipping {problem.name} with {solver} at {settings} " + f"as it is known to take {timeout} > {time_limit} seconds..." + ) + return True + return False