diff --git a/docs/poli-docs/using_poli/the_basics/defining_an_observer.ipynb b/docs/poli-docs/using_poli/the_basics/defining_an_observer.ipynb index 49db7fd1..a331c60d 100644 --- a/docs/poli-docs/using_poli/the_basics/defining_an_observer.ipynb +++ b/docs/poli-docs/using_poli/the_basics/defining_an_observer.ipynb @@ -34,7 +34,7 @@ "metadata": {}, "source": [ "All observers inherit from an `AbstractObserver` (which you can find on `poli/core/util/abstract_observer.py`). The abstract methods you need to overwrite are:\n", - "- `initialize_observer(problem_info: ProblemSetupInformation, caller_info: object, x0: np.ndarray, y0: np.ndarray, seed: int) -> object`, which gets called as part of the set-up of the objective function (when `objective_factory.create` is called).\n", + "- `initialize_observer(problem_info: BlackBoxInformation, caller_info: object, seed: int) -> object`, which gets called as part of the set-up of the objective function (when `objective_factory.create` is called).\n", "- `observe(x: np.ndarray, y: np.ndarray, context: object) -> None`, which gets called every time your optimization algorithms query the objective function.\n", "- `finish()`, which gets called either by the user, or by the object deletion at the end of the script." ] @@ -61,7 +61,7 @@ "source": [ "import numpy as np\n", "\n", - "from poli.core.problem_setup_information import ProblemSetupInformation\n", + "from poli.core.black_box_information import BlackBoxInformation\n", "from poli.core.util.abstract_observer import AbstractObserver\n", "\n", "class SimpleObserver(AbstractObserver):\n", @@ -70,10 +70,8 @@ "\n", " def initialize_observer(\n", " self,\n", - " problem_setup_info: ProblemSetupInformation,\n", + " problem_setup_info: BlackBoxInformation,\n", " caller_info: object,\n", - " x0: np.ndarray,\n", - " y0: np.ndarray,\n", " seed: int\n", " ) -> object:\n", " ...\n", @@ -111,7 +109,7 @@ "\n", "import numpy as np\n", "\n", - "from poli.core.problem_setup_information import ProblemSetupInformation\n", + "from poli.core.black_box_information import BlackBoxInformation\n", "from poli.core.util.abstract_observer import AbstractObserver\n", "\n", "THIS_DIR = Path().resolve()\n", @@ -131,10 +129,8 @@ " \n", " def initialize_observer(\n", " self,\n", - " problem_setup_info: ProblemSetupInformation,\n", + " problem_setup_info: BlackBoxInformation,\n", " caller_info: object,\n", - " x0: np.ndarray,\n", - " y0: np.ndarray,\n", " seed: int\n", " ) -> object:\n", "\n", @@ -145,10 +141,6 @@ " # (Recall that this caller info gets forwarded\n", " # from the objective_factory.create function)\n", " metadata[\"caller_info\"] = caller_info\n", - "\n", - " # Saving the initial evaluations and seed\n", - " metadata[\"x0\"] = x0.tolist()\n", - " metadata[\"y0\"] = y0.tolist()\n", " metadata[\"seed\"] = seed\n", "\n", " # Saving the metadata\n", @@ -193,7 +185,6 @@ "\n", "import numpy as np\n", "\n", - "from poli.core.problem_setup_information import ProblemSetupInformation\n", "from poli.core.util.abstract_observer import AbstractObserver\n", "\n", "THIS_DIR = Path().resolve()\n", @@ -238,7 +229,7 @@ "\n", "import numpy as np\n", "\n", - "from poli.core.problem_setup_information import ProblemSetupInformation\n", + "from poli.core.black_box_information import BlackBoxInformation\n", "from poli.core.util.abstract_observer import AbstractObserver\n", "\n", "THIS_DIR = Path().resolve()\n", @@ -258,10 +249,8 @@ " \n", " def initialize_observer(\n", " self,\n", - " problem_setup_info: ProblemSetupInformation,\n", + " problem_setup_info: BlackBoxInformation,\n", " caller_info: object,\n", - " x0: np.ndarray,\n", - " y0: np.ndarray,\n", " seed: int\n", " ) -> object:\n", "\n", @@ -288,6 +277,32 @@ " fp.write(f\"{x.tolist()}\\t{y.tolist()}\\n\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Registration\n", + "To profit from environment isolation, users have to register their observers (**once**)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from poli.core.registry import register_observer\n", + "\n", + "from simple_observer import SimpleObserver\n", + "\n", + "register_observer(\n", + " observer=SimpleObserver(),\n", + " # conda_environment_location=\"poli\", # when not providing the environment, we use the current one\n", + " observer_name=\"simple_observer\",\n", + " set_as_default_observer=False, # this is True by default!\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -304,12 +319,8 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "tags": [ - "hide-output" - ] - }, + "execution_count": 6, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -323,13 +334,10 @@ "source": [ "from poli import objective_factory\n", "\n", - "# We create an instance of the observer\n", - "observer = SimpleObserver()\n", - "\n", "# We instantiate the objective function\n", "problem = objective_factory.create(\n", " name=\"aloha\",\n", - " observer=observer,\n", + " observer_name=\"simple_observer\", # instantiate the registered observer\n", ")\n", "f, x0 = problem.black_box, problem.x0" ] @@ -338,82 +346,104 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "At this point, the observer `__init__` call created a folder called `results` right next to this file, and we can load up the metadata just to be sure:" + "Let's query the objective function at three points, and check whether the results were saved accordingly: " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'name': 'aloha', 'max_sequence_length': 5, 'aligned': True, 'fixed_length': True, 'deterministic': True, 'discrete': True, 'fidelity': None, 'alphabet': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], 'log_transform_recommended': False, 'padding_token': '', 'caller_info': None, 'x0': [['A', 'L', 'O', 'O', 'F']], 'y0': [[3]], 'seed': None}\n" + "[[0]]\n", + "[[1]]\n", + "[[5]]\n" ] } ], "source": [ - "with open(observer.experiment_path / \"metadata.json\") as fp:\n", - " print(json.load(fp))" + "print(f(np.array([list(\"MIGUE\")])))\n", + "print(f(np.array([list(\"FLEAS\")])))\n", + "print(f(np.array([list(\"ALOHA\")])))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let's query the objective function at three points, and check whether the results were saved accordingly: " + "We can verify by loading up and printing the `results.txt` file:" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[[0]]\n", - "[[1]]\n", - "[[5]]\n" + "[['M', 'I', 'G', 'U', 'E']]\t[[0]]\n", + "[['F', 'L', 'E', 'A', 'S']]\t[[1]]\n", + "[['A', 'L', 'O', 'H', 'A']]\t[[5]]\n", + "\n" ] } ], "source": [ - "print(f(np.array([list(\"MIGUE\")])))\n", - "print(f(np.array([list(\"FLEAS\")])))\n", - "print(f(np.array([list(\"ALOHA\")])))" + "with open(f.observer.experiment_path / \"results.txt\") as fp:\n", + " print(fp.read())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We can verify by loading up and printing the `results.txt` file:" + "### Discouraged Use\n", + "The example below shows what happens behind the curtains in `poli` (after dynamic instantiation). Some users may " ] }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, + "execution_count": 5, + "metadata": { + "scrolled": true, + "tags": [ + "hide-output" + ] + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[['A', 'L', 'O', 'O', 'F']]\t[[3]]\n", - "[['M', 'I', 'G', 'U', 'E']]\t[[0]]\n", - "[['F', 'L', 'E', 'A', 'S']]\t[[1]]\n", - "[['A', 'L', 'O', 'H', 'A']]\t[[5]]\n", - "\n" + "poli 🧪: Creating the objective aloha from the repository.\n", + "poli 🧪: initializing the observer.\n" ] } ], "source": [ - "with open(observer.experiment_path / \"results.txt\") as fp:\n", - " print(fp.read())" + "from poli import objective_factory\n", + "from poli.core.registry import DEFAULT_OBSERVER_NAME\n", + "\n", + "# We create an instance of the observer\n", + "observer = SimpleObserver()\n", + "\n", + "# We instantiate the objective function\n", + "problem = objective_factory.create(\n", + " name=\"aloha\",\n", + " observer_name=DEFAULT_OBSERVER_NAME, # instantiates the default observer which does nothing\n", + ")\n", + "f, x0 = problem.black_box, problem.x0\n", + "\n", + "# Then we initialize our observer...\n", + "observer.initialize_observer(seed=0, problem_setup_info=problem.black_box_information, caller_info=dict())\n", + "\n", + "#... and register it with the blackbox.\n", + "f.set_observer(observer)" ] }, { @@ -436,16 +466,13 @@ ":::{tip}\n", "\n", "If you are interested in using more complex logic for your logging, you can check the `examples` folder in `poli`, as they include two observers using `mlflow` and `wandb`.\n", - "\n", - "`poli` is also able to isolate observers, and the `examples` folder also includes a description of how.\n", - "\n", ":::\n" ] } ], "metadata": { "kernelspec": { - "display_name": "poli-base", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -459,10 +486,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" - }, - "orig_nbformat": 4 + "version": "3.9.19" + } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/docs/poli-docs/using_poli/the_basics/simple_observer.py b/docs/poli-docs/using_poli/the_basics/simple_observer.py new file mode 100644 index 00000000..4a058699 --- /dev/null +++ b/docs/poli-docs/using_poli/the_basics/simple_observer.py @@ -0,0 +1,49 @@ +from pathlib import Path +from uuid import uuid4 +import json + +import numpy as np + +from poli.core.black_box_information import BlackBoxInformation +from poli.core.util.abstract_observer import AbstractObserver + +THIS_DIR = Path().resolve() + +class SimpleObserver(AbstractObserver): + def __init__(self): + # Creating a unique id for this experiment in + # particular: + experiment_id = str(uuid4()) + self.experiment_id = experiment_id + + # Creating a local directory for the results + experiment_path = THIS_DIR / "results" / experiment_id + experiment_path.mkdir(exist_ok=True, parents=True) + + self.experiment_path = experiment_path + + def initialize_observer( + self, + problem_setup_info: BlackBoxInformation, + caller_info: object, + seed: int + ) -> object: + + # Saving the metadata for this experiment + metadata = problem_setup_info.as_dict() + + # Adding the information the user wanted to provide + # (Recall that this caller info gets forwarded + # from the objective_factory.create function) + metadata["caller_info"] = caller_info + + metadata["seed"] = seed + + # Saving the metadata + with open(self.experiment_path / "metadata.json", "w") as f: + json.dump(metadata, f) + + def observe(self, x: np.ndarray, y: np.ndarray, context=None) -> None: + # Appending these results to the results file. + with open(self.experiment_path / "results.txt", "a") as fp: + fp.write(f"{x.tolist()}\t{y.tolist()}\n")