From 862c5675d995691aff61ac75d3905b84c150bbbe Mon Sep 17 00:00:00 2001 From: TMaesing <140709195+TMaesing@users.noreply.github.com> Date: Sun, 15 Oct 2023 21:11:39 +0200 Subject: [PATCH] Unittests (#4) * Added Unittests for OAT on lpg * Adapted Unittests * Added Unittest for lpg OAT to workflow --- .github/workflows/python-unittest.yml | 1 + up_ac/tests/test_enhsp_Irace.py | 12 +-- up_ac/tests/test_fast-downward_Smac.py | 16 ++-- up_ac/tests/test_lpg_OAT.py | 128 +++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 up_ac/tests/test_lpg_OAT.py diff --git a/.github/workflows/python-unittest.yml b/.github/workflows/python-unittest.yml index 72f4067..addd632 100644 --- a/.github/workflows/python-unittest.yml +++ b/.github/workflows/python-unittest.yml @@ -73,3 +73,4 @@ jobs: python -m unittest up_ac/tests/test_allowed_combinations.py python -m unittest up_ac/tests/test_enhsp_Irace.py python -m unittest up_ac/tests/test_fast-downward_Smac.py + python -m unittest up_ac/tests/test_lpg_OAT.py diff --git a/up_ac/tests/test_enhsp_Irace.py b/up_ac/tests/test_enhsp_Irace.py index d38fd07..b0c0c38 100644 --- a/up_ac/tests/test_enhsp_Irace.py +++ b/up_ac/tests/test_enhsp_Irace.py @@ -44,10 +44,10 @@ class TestIraceEnhspOnQuality(unittest.TestCase): IAC_fb_func = IAC.get_feedback_function(igaci, engine[0], metric, 'OneshotPlanner') - def test_fb_func(self, IAC_fb_func=IAC_fb_func): + def test_A_fb_func(self, IAC_fb_func=IAC_fb_func): self.assertIsNotNone(IAC_fb_func, "Operational mode not supported") - def test_enhsp(self, IAC=IAC, + def test_A_enhsp(self, IAC=IAC, IAC_fb_func=IAC_fb_func, igaci=igaci, metric=metric, engine=engine, instances=instances): @@ -56,7 +56,7 @@ def test_enhsp(self, IAC=IAC, experiment = {'id.instance': 1, 'configuration': default_config} IAC_fb_func(experiment, IAC.scenario) - def test_evaluate(self, IAC=IAC, instances=instances, igaci=igaci, + def test_A_evaluate(self, IAC=IAC, instances=instances, igaci=igaci, engine=engine, metric=metric): if (IAC.incumbent is None) or (len(instances) == 0): @@ -96,10 +96,10 @@ class TestIraceEnhspOnRuntime(unittest.TestCase): IAC_fb_func = IAC.get_feedback_function(igaci, engine[0], metric, 'OneshotPlanner') - def test_fb_func(self, IAC_fb_func=IAC_fb_func): + def test_A_fb_func(self, IAC_fb_func=IAC_fb_func): self.assertIsNotNone(IAC_fb_func, "Operational mode not supported") - def test_enhsp(self, IAC=IAC, + def test_A_enhsp(self, IAC=IAC, IAC_fb_func=IAC_fb_func, igaci=igaci, metric=metric, engine=engine, instances=instances): @@ -108,7 +108,7 @@ def test_enhsp(self, IAC=IAC, experiment = {'id.instance': 1, 'configuration': default_config} IAC_fb_func(experiment, IAC.scenario) - def test_evaluate(self, IAC=IAC, instances=instances, igaci=igaci, + def test_A_evaluate(self, IAC=IAC, instances=instances, igaci=igaci, engine=engine, metric=metric): if (IAC.incumbent is None) or (len(instances) == 0): diff --git a/up_ac/tests/test_fast-downward_Smac.py b/up_ac/tests/test_fast-downward_Smac.py index 7c8e890..b517a96 100644 --- a/up_ac/tests/test_fast-downward_Smac.py +++ b/up_ac/tests/test_fast-downward_Smac.py @@ -62,7 +62,7 @@ class TestSmacFastDownwardOnQuality(unittest.TestCase): default_config = \ sgaci.engine_param_spaces[engine[0]].get_default_configuration() - def test_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, + def test_A_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, engine=engine, instances=instances, default_config=default_config): @@ -70,17 +70,17 @@ def test_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, self.assertIsNotNone(SAC_fb_func(default_config, instances[0], 0, reader)) - def test_optimize(self, SAC=SAC, SAC_fb_func=SAC_fb_func, + def test_B_optimize(self, SAC=SAC, SAC_fb_func=SAC_fb_func, default_config=default_config): incumbent, _ = SAC.optimize(feedback_function=SAC_fb_func) self.assertIsInstance(incumbent, dict) self.assertNotEqual(incumbent, default_config) - def test_evaluate(self, metric=metric, engine=engine, + def test_C_evaluate(self, metric=metric, engine=engine, SAC=SAC, sgaci=sgaci): perf = SAC.evaluate(metric, engine[0], 'OneshotPlanner', SAC.incumbent, sgaci, planner_timelimit=5) - self.assertIsNone(perf) + self.assertIsInstance(perf, float) class TestSmacFastDownwardOnRuntime(unittest.TestCase): @@ -127,7 +127,7 @@ class TestSmacFastDownwardOnRuntime(unittest.TestCase): default_config = \ sgaci.engine_param_spaces[engine[0]].get_default_configuration() - def test_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, + def test_A_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, engine=engine, instances=instances, default_config=default_config): @@ -135,17 +135,17 @@ def test_fb_func(self, SAC_fb_func=SAC_fb_func, sgaci=sgaci, self.assertIsNotNone(SAC_fb_func(default_config, instances[0], 0, reader)) - def test_optimize(self, SAC=SAC, SAC_fb_func=SAC_fb_func, + def test_B_optimize(self, SAC=SAC, SAC_fb_func=SAC_fb_func, default_config=default_config): incumbent, _ = SAC.optimize(feedback_function=SAC_fb_func) self.assertIsInstance(incumbent, dict) self.assertNotEqual(incumbent, default_config) - def test_evaluate(self, metric=metric, engine=engine, + def test_C_evaluate(self, metric=metric, engine=engine, SAC=SAC, sgaci=sgaci): perf = SAC.evaluate(metric, engine[0], 'OneshotPlanner', SAC.incumbent, sgaci, planner_timelimit=5) - self.assertIsNone(perf) + self.assertIsInstance(perf, float) up.shortcuts.get_environment().credits_stream = None diff --git a/up_ac/tests/test_lpg_OAT.py b/up_ac/tests/test_lpg_OAT.py new file mode 100644 index 0000000..5165d89 --- /dev/null +++ b/up_ac/tests/test_lpg_OAT.py @@ -0,0 +1,128 @@ +import unittest +import unified_planning as up +import sys +import os + +# make sure test can be run from anywhere +path = os.getcwd().rsplit('up-ac', 1)[0] +path += 'up-ac/up_ac' +if not os.path.isfile(sys.path[0] + '/configurators.py') and \ + 'up-ac' in sys.path[0]: + sys.path.insert(0, sys.path[0].rsplit('up-ac', 1)[0] + '/up-ac') + +from up_ac.OAT_configurator import OATConfigurator +from up_ac.OAT_interface import OATInterface +from up_ac.utils.download_OAT import get_OAT, delete_OAT, copy_call_engine_OAT + +get_OAT() +copy_call_engine_OAT() + + +class TestOatLpgOnQuality(unittest.TestCase): + + get_OAT() + copy_call_engine_OAT() + # pddl instance to test with + instances = [f'{path}/test_problems/visit_precedence/problem.pddl', + f'{path}/test_problems/counters/problem.pddl', + f'{path}/test_problems/depot/problem.pddl', + f'{path}/test_problems/miconic/problem.pddl', + f'{path}/test_problems/matchcellar/problem.pddl'] + + # test setting + engine = ['lpg'] + metric = "quality" + + ogaci = OATInterface() + ogaci.read_engine_pcs(engine, f'{path}/engine_pcs') + + OAC = OATConfigurator() + OAC.set_training_instance_set(instances) + OAC.set_test_instance_set(instances) + + OAC.set_scenario(engine[0], + ogaci.engine_param_spaces[engine[0]], ogaci, + configuration_time=30, n_trials=30, + crash_cost=0, planner_timelimit=15, n_workers=3, + instance_features=None, popSize=5, metric=metric, + evlaLimit=1) + + OAC_fb_func = OAC.get_feedback_function(ogaci, engine[0], + metric, 'OneshotPlanner') + + def test_A_fb_func(self, OAC_fb_func=OAC_fb_func): + self.assertIsNotNone(OAC_fb_func, "Operational mode not supported") + + def test_B_optimize(self, OAC=OAC, OAC_fb_func=OAC_fb_func): + incumbent, _ = OAC.optimize(feedback_function=OAC_fb_func) + self.assertIsInstance(incumbent, dict) + + def test_C_evaluate(self, OAC=OAC, metric=metric, + engine=engine, ogaci=ogaci): + + perf = OAC.evaluate(metric, engine[0], 'OneshotPlanner', + OAC.incumbent, ogaci) + if OAC.incumbent is None: + self.assertIsNone(perf, "No incumbent and/or no instances should return None") + else: + self.assertIsNotNone(perf, "Should have evaluated") + + +class TestOatLpgOnRuntime(unittest.TestCase): + # pddl instance to test with + instances = [f'{path}/test_problems/visit_precedence/problem.pddl', + f'{path}/test_problems/counters/problem.pddl', + f'{path}/test_problems/depot/problem.pddl', + f'{path}/test_problems/miconic/problem.pddl', + f'{path}/test_problems/matchcellar/problem.pddl'] + + # test setting + engine = ['lpg'] + metric = "runtime" + + ogaci = OATInterface() + ogaci.read_engine_pcs(engine, f'{path}/engine_pcs') + + OAC = OATConfigurator() + OAC.set_training_instance_set(instances) + OAC.set_test_instance_set(instances) + + OAC.set_scenario(engine[0], + ogaci.engine_param_spaces[engine[0]], ogaci, + configuration_time=30, n_trials=30, + crash_cost=0, planner_timelimit=15, n_workers=3, + instance_features=None, popSize=5, metric=metric, + evlaLimit=1) + + OAC_fb_func = OAC.get_feedback_function(ogaci, engine[0], + metric, 'OneshotPlanner') + + def test_A_fb_func(self, OAC_fb_func=OAC_fb_func): + self.assertIsNotNone(OAC_fb_func, "Operational mode not supported") + + def test_B_optimize(self, OAC=OAC, OAC_fb_func=OAC_fb_func): + incumbent, _ = OAC.optimize(feedback_function=OAC_fb_func) + self.assertIsInstance(incumbent, dict) + + def test_C_evaluate(self, OAC=OAC, metric=metric, + engine=engine, ogaci=ogaci): + + perf = OAC.evaluate(metric, engine[0], 'OneshotPlanner', + OAC.incumbent, ogaci) + if OAC.incumbent is None: + self.assertIsNone(perf, "No incumbent and/or no instances should return None") + else: + self.assertIsNotNone(perf, "Should have evaluated") + + def test_X_delete_OAT(self): + delete_OAT() + path = os.getcwd().rsplit('up_ac', 1)[0] + if path[-1] != "/": + path += "/" + path += 'up_ac' + self.assertFalse(os.path.isdir(f'{path}/OAT/')) + + +up.shortcuts.get_environment().credits_stream = None +if __name__ == '__main__': + unittest.main() \ No newline at end of file