From 68ff4ae8f25b43ba058ecb80138f289a38c80d56 Mon Sep 17 00:00:00 2001 From: Deyao Chen Date: Thu, 29 Dec 2022 00:27:59 +0800 Subject: [PATCH] refactor to use rpy2 local converter --- src/irace/__init__.py | 51 +++++++++++++++++------------------ tests/test_data_conversion.py | 9 +++++++ tests/test_dual_annealing.py | 13 +++++++-- tests/utils.py | 19 +++++++++++++ 4 files changed, 64 insertions(+), 28 deletions(-) create mode 100644 tests/utils.py diff --git a/src/irace/__init__.py b/src/irace/__init__.py index 636713b..643131c 100644 --- a/src/irace/__init__.py +++ b/src/irace/__init__.py @@ -7,28 +7,19 @@ from rpy2.robjects.packages import importr, PackageNotInstalledError import rpy2.robjects as ro from rpy2.robjects import pandas2ri,numpy2ri -numpy2ri.activate() # FIXME: This gives a deprecation warning but it is not clear how to replace it. from rpy2.robjects.conversion import localconverter from rpy2 import rinterface as ri from rpy2.rinterface_lib import na_values +from rpy2.rinterface_lib.sexp import NACharacterType -@ro.default_converter.rpy2py.register(ri.IntSexpVector) -def to_int(obj): - return [int(v) if v != na_values.NA_Integer else pd.NA for v in obj] +irace_converter = ro.default_converter + numpy2ri.converter + pandas2ri.converter -@ro.default_converter.rpy2py.register(ri.FloatSexpVector) -def to_float(obj): - return [float(v) if v != na_values.NA_Real else pd.NA for v in obj] +# FIXME: Make this the same as irace_converter. See https://github.com/auto-optimization/iracepy/issues/31. +irace_converter_hack = numpy2ri.converter + ro.default_converter -@ro.default_converter.rpy2py.register(ri.StrSexpVector) -def to_str(obj): - return [str(v) if v != na_values.NA_Character else pd.NA for v in obj] - -@ro.default_converter.rpy2py.register(ri.BoolSexpVector) -def to_bool(obj): - return [bool(v) if v != na_values.NA_Logical else pd.NA for v in obj] - -irace_converter = ro.default_converter + pandas2ri.converter +@irace_converter.rpy2py.register(NACharacterType) +def convert(o): + return None from rpy2.robjects.vectors import DataFrame, BoolVector, FloatVector, IntVector, StrVector, ListVector, IntArray, Matrix, ListSexpVector,FloatSexpVector,IntSexpVector,StrSexpVector,BoolSexpVector from rpy2.robjects.functions import SignatureTranslatedFunction @@ -46,6 +37,8 @@ def r_to_python(data): return data # TODO: get the actual Python function elif isinstance(data, np.ndarray): return data + elif isinstance(data, pd.DataFrame): + return data elif data == ri.NULL: return None elif data == na_values.NA_Character: @@ -55,11 +48,12 @@ def r_to_python(data): with localconverter(irace_converter): return ro.conversion.rpy2py(data) elif data.rclass[0] == 'list': - if isinstance(data.names, ri.NULLType): - keys = range(len(data)) - else: - keys = data.names - return OrderedDict(zip(keys, [r_to_python(elt) for elt in data])) + with localconverter(irace_converter): + if isinstance(data.names, ri.NULLType): + keys = range(len(data)) + else: + keys = data.names + return OrderedDict(zip(keys, [r_to_python(elt) for elt in data])) elif data.rclass[0] in ['numeric','logical','integer','RTYPES.INTSXP','array','RTYPES.LGLSXP']: if len(data) == 1: return data[0] @@ -90,7 +84,8 @@ def tmp_r_target_runner(experiment, scenario): (k,v) for k,v in py_experiment['configuration'].items() if not pd.isna(v) ) try: - ret = py_target_runner(py_experiment, py_scenario) + with localconverter(irace_converter_hack): + ret = py_target_runner(py_experiment, py_scenario) except: traceback.print_exc() ret = dict(error=traceback.format_exc()) @@ -112,7 +107,8 @@ def __init__(self, scenario, parameters_table, target_runner): self.scenario = scenario if 'instances' in scenario: self.scenario['instances'] = np.asarray(scenario['instances']) - self.parameters = self._pkg.readParameters(text = parameters_table, digits = scenario.get('digits', 4)) + with localconverter(irace_converter_hack): + self.parameters = self._pkg.readParameters(text = parameters_table, digits = scenario.get('digits', 4)) # IMPORTANT: We need to save this in a variable or it will be garbage # collected by Python and crash later. self.r_target_runner = make_target_runner(target_runner) @@ -130,12 +126,14 @@ def read_configurations(self, filename=None, text=None): return confs def set_initial_from_file(self, filename): - confs = self.read_configurations(filename = filename) + with localconverter(irace_converter): + confs = self.read_configurations(filename = filename) self.set_initial(confs) return confs def set_initial_from_str(self, text): - confs = self.read_configurations(text = text) + with localconverter(irace_converter): + confs = self.read_configurations(text = text) self.set_initial(confs) return confs @@ -149,7 +147,8 @@ def set_initial(self, x): def run(self): """Returns a Pandas DataFrame, one column per parameter and the row index are the configuration ID.""" self.scenario['targetRunner'] = self.r_target_runner - res = self._pkg.irace(ListVector(self.scenario), self.parameters) + with localconverter(irace_converter_hack): + res = self._pkg.irace(ListVector(self.scenario), self.parameters) with localconverter(irace_converter): res = ro.conversion.rpy2py(res) # Remove metadata columns. diff --git a/tests/test_data_conversion.py b/tests/test_data_conversion.py index 25856a4..5de0ed3 100644 --- a/tests/test_data_conversion.py +++ b/tests/test_data_conversion.py @@ -1,6 +1,8 @@ import numpy as np from irace import irace import pandas as pd +import re +from utils import PropagatingThread import json def target_runner(experiment, scenario): @@ -35,9 +37,16 @@ def test(): tuner = irace(scenario, params, target_runner) best_conf = tuner.run() print(best_conf) + for col in best_conf.columns: + assert not re.match(r'\..+\.', col) or col == '.ID.' for rowIndex, row in best_conf.iterrows(): #iterate over rows for columnIndex, v in row.items(): assert pd.isna(v) \ or isinstance(v, str) \ or isinstance(v, float) \ or isinstance(v, int) + +def test_thread(): + t = PropagatingThread(target=test) + t.start() + t.join() diff --git a/tests/test_dual_annealing.py b/tests/test_dual_annealing.py index 589f22e..67adb86 100644 --- a/tests/test_dual_annealing.py +++ b/tests/test_dual_annealing.py @@ -2,6 +2,7 @@ from irace import irace import os import pytest +from utils import PropagatingThread DIM=10 # This works even with parallel LB = [-5.12] @@ -50,7 +51,7 @@ def target_runner(experiment, scenario, lb = LB, ub = UB): parallel= parallel, # It can run in parallel ! logFile = "") -def test_run(): +def run_irace(scenario, parameters_table, target_runner): tuner = irace(scenario, parameters_table, target_runner) tuner.set_initial_from_str(default_values) best_confs = tuner.run() @@ -71,4 +72,12 @@ def test_fail_windows(): logFile = "") tuner = irace(scenario, parameters_table, target_runner) tuner.run() - \ No newline at end of file + + +def test_run(): + run_irace(scenario, parameters_table, target_runner) + +def test_run_in_thread(): + tuner_t = PropagatingThread(target=run_irace, args=(scenario, parameters_table, target_runner)) + tuner_t.start() + tuner_t.join() diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..8d33290 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,19 @@ +from threading import Thread + +class PropagatingThread(Thread): + def run(self): + self.exc = None + try: + if hasattr(self, '_Thread__target'): + # Thread uses name mangling prior to Python 3. + self.ret = self._Thread__target(*self._Thread__args, **self._Thread__kwargs) + else: + self.ret = self._target(*self._args, **self._kwargs) + except BaseException as e: + self.exc = e + + def join(self, timeout=None): + super(PropagatingThread, self).join(timeout) + if self.exc: + raise self.exc + return self.ret \ No newline at end of file