test2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-15T07:06:12.251711500Z",
     "start_time": "2023-10-15T07:06:12.191768100Z"
    }
   },
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "\n",
    "class pushButton(gym.Env):\n",
    "    def __init__(self,config=None):\n",
    "        self.current_steps = 0\n",
    "        self.grid_size = 4\n",
    "        self.action_space = gym.spaces.Discrete(5)\n",
    "        self.observation_space = gym.spaces.Discrete(5)\n",
    "        self.seed()\n",
    "        self.reset()\n",
    "\n",
    "    def seed(self, seed=None):\n",
    "        self.np_random, seed = gym.utils.seeding.np_random(seed)\n",
    "        return [seed]\n",
    "    \n",
    "    def reset(self, *, seed=None, options=None):\n",
    "        self._max_episode_steps = 100\n",
    "        self.current_steps = 0\n",
    "        self.activated_button = 0\n",
    "        return self.activated_button,{}\n",
    "    \n",
    "    def step(self,action):\n",
    "        self.current_steps+=1\n",
    "        reward = 1 if action==self.activated_button else 0\n",
    "        self.activated_button = np.random.randint(0,5)\n",
    "        return self.activated_button,reward,self.current_steps==self._max_episode_steps,False,{}\n",
    "        \n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-15T07:06:13.992086400Z",
     "start_time": "2023-10-15T07:06:13.455580100Z"
    }
   },
   "outputs": [],
   "source": [
    "from ray.tune.registry import register_env\n",
    "register_env(\"pb\", pushButton)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true,
    "ExecuteTime": {
     "start_time": "2023-10-15T07:06:13.995311700Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-15 03:06:14,952\tWARNING deprecation.py:50 -- DeprecationWarning: `DirectStepOptimizer` has been deprecated. This will raise an error in the future!\n",
      "2023-10-15 03:06:16,811\tINFO worker.py:1642 -- Started a local Ray instance.\n"
     ]
    }
   ],
   "source": [
    "import ray\n",
    "from ray import tune,air\n",
    "from ray.rllib.algorithms.algorithm_config import AlgorithmConfig\n",
    "from ray.rllib.algorithms.callbacks import MemoryTrackingCallbacks\n",
    "from ray.tune.logger import LoggerCallback\n",
    "from ray.air import session\n",
    "\n",
    "\n",
    "class RewardLoggerCallback(LoggerCallback):\n",
    "    def __init__(self) -> None:\n",
    "        self.results = []\n",
    "\n",
    "    def log_trial_result(self, iteration: int, trial, result):\n",
    "        reward = result  # assuming 'reward' is your reward key\n",
    "        self.results.append(reward)\n",
    "\n",
    "callback = RewardLoggerCallback()\n",
    "\n",
    "stopping_criterion = {\"training_iteration\": 50}\n",
    "ray.init(ignore_reinit_error=True, log_to_driver=False)\n",
    "tuner = tune.Tuner(\n",
    "    \"PPO\",\n",
    "    param_space={\n",
    "        \"env\": \"pb\",\n",
    "        \"framework\": \"torch\",\n",
    "        \"num_gpus\": 0,\n",
    "        \"num_workers\":8,\n",
    "        \"should_checkpoint\": True,\n",
    "    },\n",
    "    run_config=air.RunConfig(\n",
    "        stop=stopping_criterion,callbacks=[callback],\n",
    "        checkpoint_config=air.CheckpointConfig(checkpoint_frequency=1),\n",
    "        ),\n",
    ")\n",
    "results = tuner.fit()\n",
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "rewards = [i[\"episode_reward_mean\"] for i in callback.results]\n",
    "\n",
    "plt.plot(rewards)\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"Mean Reward\")\n",
    "plt.title(\"Reward vs. Iteration\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the best result\n",
    "best_result = results.get_best_result()\n",
    "\n",
    "# Get the best checkpoint\n",
    "best_checkpoint = best_result.checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Checkpoint(local_path=/Users/spacefarers/ray_results/PPO/PPO_pb_8767a_00000_0_2023-08-06_01-07-16/checkpoint_000050)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/spacefarers/mambaforge/envs/rl/lib/python3.10/site-packages/ray/rllib/algorithms/algorithm.py:442: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "`UnifiedLogger` will be removed in Ray 2.7.\n",
      "  return UnifiedLogger(config, logdir, loggers=None)\n",
      "/Users/spacefarers/mambaforge/envs/rl/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `JsonLogger interface is deprecated in favor of the `ray.tune.json.JsonLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "/Users/spacefarers/mambaforge/envs/rl/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `CSVLogger interface is deprecated in favor of the `ray.tune.csv.CSVLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "/Users/spacefarers/mambaforge/envs/rl/lib/python3.10/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `TBXLogger interface is deprecated in favor of the `ray.tune.tensorboardx.TBXLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "2023-08-06 01:09:26,567\tINFO algorithm.py:536 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
      "2023-08-06 01:09:28,401\tINFO worker.py:1636 -- Started a local Ray instance.\n"
     ]
    }
   ],
   "source": [
    "from ray.rllib.algorithms.algorithm import Algorithm\n",
    "register_env(\"pb\", pushButton)\n",
    "algo = Algorithm.from_checkpoint(best_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import sleep\n",
    "env = pushButton()\n",
    "obs = env.reset()[0]\n",
    "progression = []\n",
    "done = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action: 4\n",
      "obs: 1\n",
      "reward: 1\n",
      "WIN\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[34], line 12\u001B[0m\n\u001B[1;32m     10\u001B[0m     \u001B[39mprint\u001B[39m(\u001B[39m\"\u001B[39m\u001B[39mWIN\u001B[39m\u001B[39m\"\u001B[39m)\n\u001B[1;32m     11\u001B[0m lastObs\u001B[39m=\u001B[39mobs\n\u001B[0;32m---> 12\u001B[0m sleep(\u001B[39m1\u001B[39;49m)\n\u001B[1;32m     13\u001B[0m clear_output(\u001B[39mTrue\u001B[39;00m)\n\u001B[1;32m     14\u001B[0m \u001B[39m# progression.append(obs)\u001B[39;00m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "lastObs=obs\n",
    "while not done:\n",
    "    action = algo.compute_single_action(obs)\n",
    "    obs, reward, done, info,ads = env.step(action)\n",
    "    print(f\"action: {action}\")\n",
    "    print(f\"obs: {obs}\")\n",
    "    print(f\"reward: {reward}\")\n",
    "    if lastObs==action:\n",
    "        print(\"WIN\")\n",
    "    lastObs=obs\n",
    "    sleep(1)\n",
    "    clear_output(True)\n",
    "    # progression.append(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}