Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pass instances #34

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

pass instances #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions src/irace/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
from rpy2.robjects.vectors import DataFrame, BoolVector, FloatVector, IntVector, StrVector, ListVector, IntArray, Matrix, ListSexpVector,FloatSexpVector,IntSexpVector,StrSexpVector,BoolSexpVector
from rpy2.robjects.functions import SignatureTranslatedFunction
from rpy2.rinterface import RRuntimeWarning
import json


rpy2conversion = ro.conversion.get_conversion()
irace_converter = ro.default_converter + numpy2ri.converter + pandas2ri.converter
Expand Down Expand Up @@ -83,6 +85,7 @@ def tmp_r_target_runner(experiment, scenario):
py_experiment['configuration'] = OrderedDict(
(k,v) for k,v in py_experiment['configuration'].items() if not pd.isna(v)
)
py_experiment['instance'] = context['py_instances'][int(py_experiment['id.instance']) - 1]
try:
ret = context['py_target_runner'](py_experiment, py_scenario)
except:
Expand All @@ -94,7 +97,6 @@ def tmp_r_target_runner(experiment, scenario):
def check_windows(scenario):
if scenario.get('parallel', 1) != 1 and os.name == 'nt':
raise NotImplementedError('Parallel running on windows is not supported yet. Follow https://github.com/auto-optimization/iracepy/issues/16 for updates. Alternatively, use Linux or MacOS or the irace R package directly.')

class irace:
# Import irace R package
try:
Expand All @@ -107,12 +109,20 @@ class irace:

def __init__(self, scenario, parameters_table, target_runner):
self.scenario = scenario
self.instances = scenario.get('instances', None)
self.context = {}
if 'instances' in scenario:
self.scenario['instances'] = np.asarray(scenario['instances'])
self.context.update({
'py_instances': self.scenario['instances'],
})
self.scenario['instances'] = StrVector(list(map(lambda x: json.dumps(x, skipkeys=True, default=self.scenario.get('instanceObjectSerializer', lambda x: '<not serializable>')), self.scenario['instances'])))
Copy link
Contributor

@MLopez-Ibanez MLopez-Ibanez Jan 18, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean that the serializer is applied to strings and integers? Isn't that a waste?

Also, could you document with a comment what this line is doing? It seems to replace the instance with the string '<not serializable>' unless the user provides an 'instanceObjectSerializer'. Is that the case? If so, how can the user serialize their instances? It is unclear to me how all this works and will be used in practice.

A couple of examples using for example:

self.scenario.pop('instanceObjectSerializer', None)
with localconverter(irace_converter_hack):
self.parameters = self._pkg.readParameters(text = parameters_table, digits = self.scenario.get('digits', 4))
self.context = {'py_target_runner' : target_runner,
'py_scenario': self.scenario }
self.context.update({
'py_target_runner' : target_runner,
'py_scenario': self.scenario,
})
check_windows(scenario)

def read_configurations(self, filename=None, text=None):
Expand Down
69 changes: 68 additions & 1 deletion tests/test_data_passable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from irace import irace
import pandas as pd
from multiprocessing import Queue
import pytest
import os

q = Queue()

Expand All @@ -12,6 +14,10 @@ def target_runner(experiment, scenario):
else:
return dict(cost=1)

def target_runner2(experiment, scenario):
if experiment['id.instance'] == 1:
experiment['instance'].put(1335)
return dict(cost=1)

params = '''
one "" c ('0', '1')
Expand All @@ -34,4 +40,65 @@ def test():
tuner = irace(scenario, params, target_runner)
best_conf = tuner.run()
assert q.get() == 124


def test_instances():
q = Queue()
scenario = dict(
instances = [q],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why would an instance be a Queue? What is this testing? Could you add a testcase that is more realistic, like instances being a list of functions that need to me minimized?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's because Queue is not serializable and can only work if the queue on the other hand shares the same memory address.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But why pass it in "instances" and not via other way? You are using "instances" for something that is not its purpose.
Does this need to be part of the scenario?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess it doesn't need to be. But I think it's nice if there are multiple ways to pass stuff around.

maxExperiments = 180,
debugLevel = 0,
parallel = 1,
logFile = "",
seed = 123
)
tuner = irace(scenario, params, target_runner2)
best_conf = tuner.run()
assert q.get() == 1335

@pytest.mark.skipif(os.name == 'nt',
reason="Parallel on Windows not supported")
def test_instances2():
q = Queue()
scenario = dict(
instances = [q],
maxExperiments = 180,
debugLevel = 0,
parallel = 2,
logFile = "",
seed = 123
)
tuner = irace(scenario, params, target_runner2)
best_conf = tuner.run()
assert q.get() == 1335

def test_default_serializer():
q = Queue()
scenario = dict(
instances = [q],
maxExperiments = 180,
debugLevel = 0,
parallel = 1,
logFile = "",
seed = 123,
instanceObjectSerializer = lambda x: 'hello world'
)
tuner = irace(scenario, params, target_runner2)
best_conf = tuner.run()
assert q.get() == 1335

@pytest.mark.skipif(os.name == 'nt',
reason="Parallel on Windows not supported")
def test_default_serializer():
q = Queue()
scenario = dict(
instances = [q],
maxExperiments = 180,
debugLevel = 0,
parallel = 2,
logFile = "",
seed = 123,
instanceObjectSerializer = lambda x: 'hello world'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is this doing? Could you document it somewhere?

Irace is extremely well documented and most of its documentation currently applies to iracepy. See: https://mlopez-ibanez.github.io/irace/ and https://mlopez-ibanez.github.io/irace/irace-package.pdf

)
tuner = irace(scenario, params, target_runner2)
best_conf = tuner.run()
assert q.get() == 1335