Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updates the observer documentation #19

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

updates the observer documentation #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 85 additions & 59 deletions docs/poli-docs/using_poli/the_basics/defining_an_observer.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
"metadata": {},
"source": [
"All observers inherit from an `AbstractObserver` (which you can find on `poli/core/util/abstract_observer.py`). The abstract methods you need to overwrite are:\n",
"- `initialize_observer(problem_info: ProblemSetupInformation, caller_info: object, x0: np.ndarray, y0: np.ndarray, seed: int) -> object`, which gets called as part of the set-up of the objective function (when `objective_factory.create` is called).\n",
"- `initialize_observer(problem_info: BlackBoxInformation, caller_info: object, seed: int) -> object`, which gets called as part of the set-up of the objective function (when `objective_factory.create` is called).\n",
"- `observe(x: np.ndarray, y: np.ndarray, context: object) -> None`, which gets called every time your optimization algorithms query the objective function.\n",
"- `finish()`, which gets called either by the user, or by the object deletion at the end of the script."
]
Expand All @@ -61,7 +61,7 @@
"source": [
"import numpy as np\n",
"\n",
"from poli.core.problem_setup_information import ProblemSetupInformation\n",
"from poli.core.black_box_information import BlackBoxInformation\n",
"from poli.core.util.abstract_observer import AbstractObserver\n",
"\n",
"class SimpleObserver(AbstractObserver):\n",
Expand All @@ -70,10 +70,8 @@
"\n",
" def initialize_observer(\n",
" self,\n",
" problem_setup_info: ProblemSetupInformation,\n",
" problem_setup_info: BlackBoxInformation,\n",
" caller_info: object,\n",
" x0: np.ndarray,\n",
" y0: np.ndarray,\n",
" seed: int\n",
" ) -> object:\n",
" ...\n",
Expand Down Expand Up @@ -111,7 +109,7 @@
"\n",
"import numpy as np\n",
"\n",
"from poli.core.problem_setup_information import ProblemSetupInformation\n",
"from poli.core.black_box_information import BlackBoxInformation\n",
"from poli.core.util.abstract_observer import AbstractObserver\n",
"\n",
"THIS_DIR = Path().resolve()\n",
Expand All @@ -131,10 +129,8 @@
" \n",
" def initialize_observer(\n",
" self,\n",
" problem_setup_info: ProblemSetupInformation,\n",
" problem_setup_info: BlackBoxInformation,\n",
" caller_info: object,\n",
" x0: np.ndarray,\n",
" y0: np.ndarray,\n",
" seed: int\n",
" ) -> object:\n",
"\n",
Expand All @@ -145,10 +141,6 @@
" # (Recall that this caller info gets forwarded\n",
" # from the objective_factory.create function)\n",
" metadata[\"caller_info\"] = caller_info\n",
"\n",
" # Saving the initial evaluations and seed\n",
" metadata[\"x0\"] = x0.tolist()\n",
" metadata[\"y0\"] = y0.tolist()\n",
" metadata[\"seed\"] = seed\n",
"\n",
" # Saving the metadata\n",
Expand Down Expand Up @@ -193,7 +185,6 @@
"\n",
"import numpy as np\n",
"\n",
"from poli.core.problem_setup_information import ProblemSetupInformation\n",
"from poli.core.util.abstract_observer import AbstractObserver\n",
"\n",
"THIS_DIR = Path().resolve()\n",
Expand Down Expand Up @@ -238,7 +229,7 @@
"\n",
"import numpy as np\n",
"\n",
"from poli.core.problem_setup_information import ProblemSetupInformation\n",
"from poli.core.black_box_information import BlackBoxInformation\n",
"from poli.core.util.abstract_observer import AbstractObserver\n",
"\n",
"THIS_DIR = Path().resolve()\n",
Expand All @@ -258,10 +249,8 @@
" \n",
" def initialize_observer(\n",
" self,\n",
" problem_setup_info: ProblemSetupInformation,\n",
" problem_setup_info: BlackBoxInformation,\n",
" caller_info: object,\n",
" x0: np.ndarray,\n",
" y0: np.ndarray,\n",
" seed: int\n",
" ) -> object:\n",
"\n",
Expand All @@ -288,6 +277,32 @@
" fp.write(f\"{x.tolist()}\\t{y.tolist()}\\n\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Registration\n",
"To profit from environment isolation, users have to register their observers (**once**)."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from poli.core.registry import register_observer\n",
"\n",
"from simple_observer import SimpleObserver\n",
"\n",
"register_observer(\n",
" observer=SimpleObserver(),\n",
" # conda_environment_location=\"poli\", # when not providing the environment, we use the current one\n",
" observer_name=\"simple_observer\",\n",
" set_as_default_observer=False, # this is True by default!\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -304,12 +319,8 @@
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"tags": [
"hide-output"
]
},
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
Expand All @@ -323,13 +334,10 @@
"source": [
"from poli import objective_factory\n",
"\n",
"# We create an instance of the observer\n",
"observer = SimpleObserver()\n",
"\n",
"# We instantiate the objective function\n",
"problem = objective_factory.create(\n",
" name=\"aloha\",\n",
" observer=observer,\n",
" observer_name=\"simple_observer\", # instantiate the registered observer\n",
")\n",
"f, x0 = problem.black_box, problem.x0"
]
Expand All @@ -338,82 +346,104 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"At this point, the observer `__init__` call created a folder called `results` right next to this file, and we can load up the metadata just to be sure:"
"Let's query the objective function at three points, and check whether the results were saved accordingly: "
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'name': 'aloha', 'max_sequence_length': 5, 'aligned': True, 'fixed_length': True, 'deterministic': True, 'discrete': True, 'fidelity': None, 'alphabet': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], 'log_transform_recommended': False, 'padding_token': '', 'caller_info': None, 'x0': [['A', 'L', 'O', 'O', 'F']], 'y0': [[3]], 'seed': None}\n"
"[[0]]\n",
"[[1]]\n",
"[[5]]\n"
]
}
],
"source": [
"with open(observer.experiment_path / \"metadata.json\") as fp:\n",
" print(json.load(fp))"
"print(f(np.array([list(\"MIGUE\")])))\n",
"print(f(np.array([list(\"FLEAS\")])))\n",
"print(f(np.array([list(\"ALOHA\")])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's query the objective function at three points, and check whether the results were saved accordingly: "
"We can verify by loading up and printing the `results.txt` file:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[0]]\n",
"[[1]]\n",
"[[5]]\n"
"[['M', 'I', 'G', 'U', 'E']]\t[[0]]\n",
"[['F', 'L', 'E', 'A', 'S']]\t[[1]]\n",
"[['A', 'L', 'O', 'H', 'A']]\t[[5]]\n",
"\n"
]
}
],
"source": [
"print(f(np.array([list(\"MIGUE\")])))\n",
"print(f(np.array([list(\"FLEAS\")])))\n",
"print(f(np.array([list(\"ALOHA\")])))"
"with open(f.observer.experiment_path / \"results.txt\") as fp:\n",
" print(fp.read())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can verify by loading up and printing the `results.txt` file:"
"### Discouraged Use\n",
"The example below shows what happens behind the curtains in `poli` (after dynamic instantiation). Some users may "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"execution_count": 5,
"metadata": {
"scrolled": true,
"tags": [
"hide-output"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[['A', 'L', 'O', 'O', 'F']]\t[[3]]\n",
"[['M', 'I', 'G', 'U', 'E']]\t[[0]]\n",
"[['F', 'L', 'E', 'A', 'S']]\t[[1]]\n",
"[['A', 'L', 'O', 'H', 'A']]\t[[5]]\n",
"\n"
"poli 🧪: Creating the objective aloha from the repository.\n",
"poli 🧪: initializing the observer.\n"
]
}
],
"source": [
"with open(observer.experiment_path / \"results.txt\") as fp:\n",
" print(fp.read())"
"from poli import objective_factory\n",
"from poli.core.registry import DEFAULT_OBSERVER_NAME\n",
"\n",
"# We create an instance of the observer\n",
"observer = SimpleObserver()\n",
"\n",
"# We instantiate the objective function\n",
"problem = objective_factory.create(\n",
" name=\"aloha\",\n",
" observer_name=DEFAULT_OBSERVER_NAME, # instantiates the default observer which does nothing\n",
")\n",
"f, x0 = problem.black_box, problem.x0\n",
"\n",
"# Then we initialize our observer...\n",
"observer.initialize_observer(seed=0, problem_setup_info=problem.black_box_information, caller_info=dict())\n",
"\n",
"#... and register it with the blackbox.\n",
"f.set_observer(observer)"
]
},
{
Expand All @@ -436,16 +466,13 @@
":::{tip}\n",
"\n",
"If you are interested in using more complex logic for your logging, you can check the `examples` folder in `poli`, as they include two observers using `mlflow` and `wandb`.\n",
"\n",
"`poli` is also able to isolate observers, and the `examples` folder also includes a description of how.\n",
"\n",
":::\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "poli-base",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -459,10 +486,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
},
"orig_nbformat": 4
"version": "3.9.19"
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
49 changes: 49 additions & 0 deletions docs/poli-docs/using_poli/the_basics/simple_observer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from pathlib import Path
from uuid import uuid4
import json

import numpy as np

from poli.core.black_box_information import BlackBoxInformation
from poli.core.util.abstract_observer import AbstractObserver

THIS_DIR = Path().resolve()

class SimpleObserver(AbstractObserver):
def __init__(self):
# Creating a unique id for this experiment in
# particular:
experiment_id = str(uuid4())
self.experiment_id = experiment_id

# Creating a local directory for the results
experiment_path = THIS_DIR / "results" / experiment_id
experiment_path.mkdir(exist_ok=True, parents=True)

self.experiment_path = experiment_path

def initialize_observer(
self,
problem_setup_info: BlackBoxInformation,
caller_info: object,
seed: int
) -> object:

# Saving the metadata for this experiment
metadata = problem_setup_info.as_dict()

# Adding the information the user wanted to provide
# (Recall that this caller info gets forwarded
# from the objective_factory.create function)
metadata["caller_info"] = caller_info

metadata["seed"] = seed

# Saving the metadata
with open(self.experiment_path / "metadata.json", "w") as f:
json.dump(metadata, f)

def observe(self, x: np.ndarray, y: np.ndarray, context=None) -> None:
# Appending these results to the results file.
with open(self.experiment_path / "results.txt", "a") as fp:
fp.write(f"{x.tolist()}\t{y.tolist()}\n")