Skip to content

Commit

Permalink
Move solver issue and timeout listings to the test-set themselves
Browse files Browse the repository at this point in the history
  • Loading branch information
@stephane-caron
stephane-caron committed Dec 8, 2023
1 parent 55928ab commit 30ee2dd
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 125 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,13 @@ All notable changes to this project will be documented in this file.
### Changed

* Don't install solvers by default from PyPI
* Move solver issue and timeout listings to test-set themselves
* Rename the project "qpbenchmark"

### Removed

* Module-wide skip solver issue/timeout functions

## [1.2.0] - 2023-11-27

### Added
Expand Down
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ Contributions to improving this benchmark are welcome. You can for instance prop

## Citation

To cite this benchmark in your scientific works:
If you use `qpbenchmark` in your scientific works, please cite it *e.g.* as follows:

```bibtex
@software{qpbenchmark_2023,
Expand All @@ -148,7 +148,7 @@ To cite this benchmark in your scientific works:
}
```

You can also click on the ``Cite this repository`` button on GitHub, at the top-right of the [repository page](https://github.com/qpsolvers/qpbenchmark/).
You can also click on ``Cite this repository`` at the top-right of the [repository page](https://github.com/qpsolvers/qpbenchmark/).

## See also

Expand Down
7 changes: 4 additions & 3 deletions qpbenchmark/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from qpsolvers.exceptions import SolverNotFound

from .results import Results
from .solver_issues import skip_solver_issue, skip_solver_timeout
from .spdlog import logging
from .test_set import TestSet
from .utils import time_solve_problem
Expand Down Expand Up @@ -80,7 +79,7 @@ def run(
for solver in filtered_solvers:
for settings in filtered_settings:
time_limit = test_set.tolerances[settings].runtime
if skip_solver_issue(problem, solver):
if (problem, solver) in test_set.known_solver_issues:
failure = (
problem,
solver,
Expand All @@ -90,7 +89,9 @@ def run(
)
results.update(*failure)
continue
if skip_solver_timeout(time_limit, problem, solver, settings):
if test_set.skip_solver_timeout(
time_limit, problem, solver, settings
):
failure = (
problem,
solver,
Expand Down
119 changes: 0 additions & 119 deletions qpbenchmark/solver_issues.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,122 +16,3 @@
# limitations under the License.

"""Some solvers fail on some problems. Make sure we handle and report those."""

from typing import Dict, Tuple

from .problem import Problem
from .spdlog import logging


def skip_solver_issue(problem: Problem, solver: str) -> bool:
"""Check whether a solver is known to fail at solving a given problem.
Args:
problem: Problem to solve.
solver: QP solver.
Returns:
True if solver is known to fail at solving the problem.
"""
if solver == "proxqp" and problem.name == "QGFRDXPN":
# https://github.com/Simple-Robotics/proxsuite/issues/63
return True
if solver == "highs" and problem.name == "STADAT1":
# https://github.com/ERGO-Code/HiGHS/issues/995
return True
return False


def skip_solver_timeout(
time_limit: float, problem: Problem, solver: str, settings: str
) -> bool:
"""Skip known solver timeouts.
Args:
time_limit: Time limit in seconds.
problem: Problem to solve.
solver: QP solver.
settings: QP solver settings.
Note:
This function only checks for timeouts that the solvers are not able to
handle by themselves, e.g. for those who do not provide a time limit
parameter.
Returns:
True if `solver` is known to take more than `time_limit` seconds on
`problem` when using the solver parameters defined in `settings`.
"""
minutes = 60.0
# TODO: these timeouts should be moved to test sets themselves
known_timeout_settings: Dict[Tuple[str, str, str], float] = {
("AUG2D", "highs", "*"): 40 * minutes,
("AUG2D", "proxqp", "high_accuracy"): 40 * minutes,
("AUG2DC", "highs", "*"): 40 * minutes,
("AUG2DC", "proxqp", "high_accuracy"): 40 * minutes,
("AUG2DCQP", "proxqp", "high_accuracy"): 20 * minutes,
("AUG2DQP", "proxqp", "high_accuracy"): 30 * minutes,
("BOYD1", "proxqp", "*"): 30 * minutes,
("BOYD2", "cvxopt", "mid_accuracy"): 30 * minutes,
("BOYD2", "proxqp", "*"): 20 * minutes,
("CONT-101", "proxqp", "*"): 30 * minutes,
("CONT-200", "proxqp", "*"): 20 * minutes,
("CONT-201", "proxqp", "*"): 30 * minutes,
("CONT-300", "cvxopt", "*"): 20 * minutes,
("CONT-300", "highs", "default"): 30 * minutes,
("CONT-300", "highs", "high_accuracy"): 30 * minutes,
("CONT-300", "proxqp", "*"): 60 * minutes,
("CVXQP1_L", "proxqp", "*"): 20 * minutes,
("CVXQP2_L", "proxqp", "high_accuracy"): 30 * minutes,
("CVXQP3_L", "cvxopt", "*"): 20 * minutes,
("CVXQP3_L", "proxqp", "*"): 30 * minutes,
("DTOC3", "proxqp", "high_accuracy"): 20 * minutes,
("DTOC3", "proxqp", "low_accuracy"): 20 * minutes,
("DTOC3", "proxqp", "mid_accuracy"): 20 * minutes,
("EXDATA", "proxqp", "*"): 30 * minutes,
("LISWET1", "proxqp", "*"): 20 * minutes,
("LISWET10", "proxqp", "*"): 50 * minutes,
("LISWET11", "proxqp", "high_accuracy"): 40 * minutes,
("LISWET11", "proxqp", "mid_accuracy"): 30 * minutes,
("LISWET12", "proxqp", "high_accuracy"): 20 * minutes,
("LISWET12", "proxqp", "low_accuracy"): 60 * minutes,
("LISWET12", "proxqp", "mid_accuracy"): 60 * minutes,
("LISWET2", "proxqp", "high_accuracy"): 20 * minutes,
("LISWET2", "proxqp", "mid_accuracy"): 20 * minutes,
("LISWET3", "proxqp", "high_accuracy"): 30 * minutes,
("LISWET4", "proxqp", "high_accuracy"): 20 * minutes,
("LISWET4", "proxqp", "mid_accuracy"): 30 * minutes,
("LISWET5", "proxqp", "high_accuracy"): 20 * minutes,
("LISWET6", "proxqp", "high_accuracy"): 20 * minutes,
("LISWET6", "proxqp", "mid_accuracy"): 20 * minutes,
("LISWET7", "proxqp", "high_accuracy"): 30 * minutes,
("LISWET7", "proxqp", "mid_accuracy"): 30 * minutes,
("LISWET8", "proxqp", "high_accuracy"): 30 * minutes,
("LISWET8", "proxqp", "mid_accuracy"): 30 * minutes,
("LISWET9", "proxqp", "high_accuracy"): 30 * minutes,
("LISWET9", "proxqp", "low_accuracy"): 30 * minutes,
("LISWET9", "proxqp", "mid_accuracy"): 30 * minutes,
("POWELL20", "proxqp", "*"): 30 * minutes,
("QGFRDXPN", "proxqp", "*"): 20 * minutes,
("QSHIP08L", "proxqp", "*"): 20 * minutes,
("QSHIP12L", "proxqp", "*"): 20 * minutes,
("STADAT1", "proxqp", "*"): 20 * minutes,
("STADAT2", "proxqp", "*"): 20 * minutes,
("STADAT3", "proxqp", "*"): 20 * minutes,
("UBH1", "proxqp", "*"): 20 * minutes,
("YAO", "proxqp", "*"): 20 * minutes,
}
timeout = (
known_timeout_settings[(problem.name, solver, settings)]
if (problem.name, solver, settings) in known_timeout_settings
else known_timeout_settings[(problem.name, solver, "*")]
if (problem.name, solver, "*") in known_timeout_settings
else 0.0
)
if timeout > time_limit:
logging.warning(
f"Skipping {problem.name} with {solver} at {settings} "
f"as it is known to take {timeout} > {time_limit} seconds..."
)
return True
return False
41 changes: 40 additions & 1 deletion qpbenchmark/test_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"""Base class for test sets."""

import abc
from typing import Dict, Iterator, Optional
from typing import Dict, Iterator, Optional, Set, Tuple

import qpsolvers

Expand All @@ -39,6 +39,8 @@ class TestSet(abc.ABC):
tolerances: Validation tolerances.
"""

known_solver_issues: Set[Tuple[str, str]]
known_solver_timeouts: Dict[Tuple[str, str, str], float]
solver_settings: Dict[str, SolverSettings]
tolerances: Dict[str, Tolerance]

Expand Down Expand Up @@ -136,6 +138,8 @@ def __init__(self):
f"Solver '{solver}' is available but skipped "
"as its settings are unknown"
)
self.known_solver_issues = set()
self.known_solver_timeouts = {}
self.solver_settings = {}
self.solvers = solvers
self.tolerances = {}
Expand Down Expand Up @@ -193,3 +197,38 @@ def get_problem(self, name: str) -> Optional[Problem]:
f"problem '{name}' not found "
f"in the {self.__class__.__name__} test set"
)

def skip_solver_timeout(
self, time_limit: float, problem: Problem, solver: str, settings: str
) -> bool:
"""Skip known solver timeouts.
Args:
time_limit: Time limit in seconds.
problem: Problem to solve.
solver: QP solver.
settings: QP solver settings.
Note:
This function only checks for timeouts that the solvers are not
able to handle by themselves, e.g. for those who do not provide a
time limit parameter.
Returns:
True if `solver` is known to take more than `time_limit` seconds on
`problem` when using the solver parameters defined in `settings`.
"""
timeout = (
self.known_solver_timeouts[(problem.name, solver, settings)]
if (problem.name, solver, settings) in self.known_solver_timeouts
else self.known_solver_timeouts[(problem.name, solver, "*")]
if (problem.name, solver, "*") in self.known_solver_timeouts
else 0.0
)
if timeout > time_limit:
logging.warning(
f"Skipping {problem.name} with {solver} at {settings} "
f"as it is known to take {timeout} > {time_limit} seconds..."
)
return True
return False

0 comments on commit 30ee2dd

Please sign in to comment.