testApi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"This is a minimal example of using Tianshou with MARL to train agents.\n",
    "\n",
    "Author: Will (https://github.com/WillDudley)\n",
    "\n",
    "Python version used: 3.8.10\n",
    "\n",
    "Requirements:\n",
    "pettingzoo == 1.22.0\n",
    "git+https://github.com/thu-ml/tianshou\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "from typing import Optional, Tuple\n",
    "\n",
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "import torch\n",
    "from tianshou.data import Collector, VectorReplayBuffer\n",
    "from tianshou.env import DummyVectorEnv\n",
    "from tianshou.env.pettingzoo_env import PettingZooEnv\n",
    "from tianshou.policy import BasePolicy, DQNPolicy, MultiAgentPolicyManager, RandomPolicy\n",
    "from tianshou.trainer import offpolicy_trainer\n",
    "from tianshou.utils.net.common import Net\n",
    "\n",
    "from pettingzoo.classic import tictactoe_v3\n",
    "\n",
    "\n",
    "def _get_agents(\n",
    "    agent_learn: Optional[BasePolicy] = None,\n",
    "    agent_opponent: Optional[BasePolicy] = None,\n",
    "    optim: Optional[torch.optim.Optimizer] = None,\n",
    ") -> Tuple[BasePolicy, torch.optim.Optimizer, list]:\n",
    "    env = _get_env()\n",
    "    observation_space = (\n",
    "        env.observation_space[\"observation\"]\n",
    "        if isinstance(env.observation_space, gym.spaces.Dict)\n",
    "        else env.observation_space\n",
    "    )\n",
    "    if agent_learn is None:\n",
    "        # model\n",
    "        net = Net(\n",
    "            state_shape=observation_space.shape or observation_space.n,\n",
    "            action_shape=env.action_space.shape or env.action_space.n,\n",
    "            hidden_sizes=[128, 128, 128, 128],\n",
    "            device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
    "        ).to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        if optim is None:\n",
    "            optim = torch.optim.Adam(net.parameters(), lr=1e-4)\n",
    "        agent_learn = DQNPolicy(\n",
    "            model=net,\n",
    "            optim=optim,\n",
    "            discount_factor=0.9,\n",
    "            estimation_step=3,\n",
    "            target_update_freq=320,\n",
    "        )\n",
    "\n",
    "    if agent_opponent is None:\n",
    "        agent_opponent = RandomPolicy()\n",
    "\n",
    "    agents = [agent_opponent, agent_learn]\n",
    "    policy = MultiAgentPolicyManager(agents, env)\n",
    "    return policy, optim, env.agents\n",
    "\n",
    "\n",
    "def _get_env():\n",
    "    \"\"\"This function is needed to provide callables for DummyVectorEnv.\"\"\"\n",
    "    return PettingZooEnv(tictactoe_v3.env())\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # ======== Step 1: Environment setup =========\n",
    "    train_envs = DummyVectorEnv([_get_env for _ in range(10)])\n",
    "    test_envs = DummyVectorEnv([_get_env for _ in range(10)])\n",
    "\n",
    "    # seed\n",
    "    seed = 1\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    train_envs.seed(seed)\n",
    "    test_envs.seed(seed)\n",
    "\n",
    "    # ======== Step 2: Agent setup =========\n",
    "    policy, optim, agents = _get_agents()\n",
    "\n",
    "    # ======== Step 3: Collector setup =========\n",
    "    train_collector = Collector(\n",
    "        policy,\n",
    "        train_envs,\n",
    "        VectorReplayBuffer(20_000, len(train_envs)),\n",
    "        exploration_noise=True,\n",
    "    )\n",
    "    test_collector = Collector(policy, test_envs, exploration_noise=True)\n",
    "    # policy.set_eps(1)\n",
    "    train_collector.collect(n_step=64 * 10)  # batch size * training_num\n",
    "\n",
    "    # ======== Step 4: Callback functions setup =========\n",
    "    def save_best_fn(policy):\n",
    "        model_save_path = os.path.join(\"log\", \"rps\", \"dqn\", \"policy.pth\")\n",
    "        os.makedirs(os.path.join(\"log\", \"rps\", \"dqn\"), exist_ok=True)\n",
    "        torch.save(policy.policies[agents[1]].state_dict(), model_save_path)\n",
    "\n",
    "    def stop_fn(mean_rewards):\n",
    "        return mean_rewards >= 0.6\n",
    "\n",
    "    def train_fn(epoch, env_step):\n",
    "        policy.policies[agents[1]].set_eps(0.1)\n",
    "\n",
    "    def test_fn(epoch, env_step):\n",
    "        policy.policies[agents[1]].set_eps(0.05)\n",
    "\n",
    "    def reward_metric(rews):\n",
    "        return rews[:, 1]\n",
    "\n",
    "    # ======== Step 5: Run the trainer =========\n",
    "    result = offpolicy_trainer(\n",
    "        policy=policy,\n",
    "        train_collector=train_collector,\n",
    "        test_collector=test_collector,\n",
    "        max_epoch=50,\n",
    "        step_per_epoch=1000,\n",
    "        step_per_collect=50,\n",
    "        episode_per_test=10,\n",
    "        batch_size=64,\n",
    "        train_fn=train_fn,\n",
    "        test_fn=test_fn,\n",
    "        stop_fn=stop_fn,\n",
    "        save_best_fn=save_best_fn,\n",
    "        update_per_step=0.1,\n",
    "        test_in_train=False,\n",
    "        reward_metric=reward_metric,\n",
    "    )\n",
    "\n",
    "    # return result, policy.policies[agents[1]]\n",
    "    print(f\"\\n==========Result==========\\n{result}\")\n",
    "    print(\"\\n(the trained policy can be accessed via policy.policies[agents[1]])\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from environment import Environment\n",
    "\n",
    "env = Environment.VehicleJobSchedulingEnvACE()\n",
    "env.reset()\n",
    "for agent in env.agent_iter(10000):\n",
    "    env.step(env.action_space(agent).sample())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jax.numpy as jnp\n",
    "from jax import grad, jit, vmap\n",
    "from jax import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "key = random.PRNGKey(0)\n",
    "x = random.normal(key, (10,))\n",
    "print(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = 3000\n",
    "x = random.normal(key, (size, size), dtype=jnp.float32)\n",
    "%timeit jnp.dot(x, x.T).block_until_ready()  # runs on the GPU\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selu(x, alpha=1.67, lmbda=1.05):\n",
    "  return lmbda * jnp.where(x > 0, x, alpha * jnp.exp(x) - alpha)\n",
    "\n",
    "x = random.normal(key, (1000000,))\n",
    "%timeit selu(x).block_until_ready()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selu_jit = jit(selu)\n",
    "%timeit selu_jit(x).block_until_ready()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'jax.numpy' has no attribute 'random'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\testApi.ipynb 单元格 8\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Yuming/Documents/ResMan/testApi.ipynb#X10sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39menvironment\u001b[39;00m \u001b[39mimport\u001b[39;00m environment_jax\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/Yuming/Documents/ResMan/testApi.ipynb#X10sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m env \u001b[39m=\u001b[39m environment_jax\u001b[39m.\u001b[39mVehicleJobSchedulingEnvACE()\n",
      "File \u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\environment\\environment_jax.py:473\u001b[0m\n\u001b[0;32m    469\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbacklog[:\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbacklog[\u001b[39m1\u001b[39m:]\n\u001b[0;32m    470\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbacklog[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m] \u001b[39m=\u001b[39m job\n\u001b[1;32m--> 473\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mJobRecord\u001b[39;00m:\n\u001b[0;32m    474\u001b[0m     \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    475\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrecord \u001b[39m=\u001b[39m {}\n",
      "File \u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\environment\\environment_jax.py:477\u001b[0m, in \u001b[0;36mJobRecord\u001b[1;34m()\u001b[0m\n\u001b[0;32m    474\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    475\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrecord \u001b[39m=\u001b[39m {}\n\u001b[1;32m--> 477\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mnew_record\u001b[39m(\u001b[39mself\u001b[39m, job\u001b[39m=\u001b[39mJob()):\n\u001b[0;32m    478\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m    479\u001b[0m \u001b[39m    Purpose: add new record\u001b[39;00m\n\u001b[0;32m    480\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m    481\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrecord[job\u001b[39m.\u001b[39mid] \u001b[39m=\u001b[39m job\n",
      "File \u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\environment\\environment_jax.py:145\u001b[0m, in \u001b[0;36mJob.__init__\u001b[1;34m(self, res_vec, job_len, job_id, enter_time, priority, average_cost_vec)\u001b[0m\n\u001b[0;32m    143\u001b[0m \u001b[39m# self.job_vec = self.generate_job()  # 生成的任务向量\u001b[39;00m\n\u001b[0;32m    144\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maverage_cost_vec \u001b[39m=\u001b[39m average_cost_vec  \u001b[39m# 平均成本向量\u001b[39;00m\n\u001b[1;32m--> 145\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbudget \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcalculate_budget(average_cost_vec)  \u001b[39m# 计算的预算\u001b[39;00m\n\u001b[0;32m    146\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpay \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m  \u001b[39m# 支付金额\u001b[39;00m\n\u001b[0;32m    147\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mutility \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\environment\\environment_jax.py:165\u001b[0m, in \u001b[0;36mJob.calculate_budget\u001b[1;34m(self, average_cost_vec, var)\u001b[0m\n\u001b[0;32m    160\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcalculate_budget\u001b[39m(\u001b[39mself\u001b[39m, average_cost_vec, var\u001b[39m=\u001b[39m\u001b[39m0.03\u001b[39m):\n\u001b[0;32m    161\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mmax\u001b[39m(\n\u001b[0;32m    162\u001b[0m         \u001b[39m0\u001b[39m,\n\u001b[0;32m    163\u001b[0m         np\u001b[39m.\u001b[39mdot(np\u001b[39m.\u001b[39marray(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mres_vec), np\u001b[39m.\u001b[39marray(average_cost_vec))\n\u001b[0;32m    164\u001b[0m         \u001b[39m*\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlen\n\u001b[1;32m--> 165\u001b[0m         \u001b[39m*\u001b[39m ((\u001b[39m1\u001b[39m \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpriority \u001b[39m/\u001b[39m \u001b[39m10\u001b[39m) \u001b[39m+\u001b[39m var \u001b[39m*\u001b[39m np\u001b[39m.\u001b[39;49mrandom\u001b[39m.\u001b[39mnormal()),\n\u001b[0;32m    166\u001b[0m     )\n",
      "File \u001b[1;32mc:\\Users\\Yuming\\Documents\\ResMan\\man\\lib\\site-packages\\jax\\_src\\deprecations.py:53\u001b[0m, in \u001b[0;36mdeprecation_getattr.<locals>.getattr\u001b[1;34m(name)\u001b[0m\n\u001b[0;32m     51\u001b[0m   warnings\u001b[39m.\u001b[39mwarn(message, \u001b[39mDeprecationWarning\u001b[39;00m, stacklevel\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m)\n\u001b[0;32m     52\u001b[0m   \u001b[39mreturn\u001b[39;00m fn\n\u001b[1;32m---> 53\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mAttributeError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mmodule \u001b[39m\u001b[39m{\u001b[39;00mmodule\u001b[39m!r}\u001b[39;00m\u001b[39m has no attribute \u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m!r}\u001b[39;00m\u001b[39m\"\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: module 'jax.numpy' has no attribute 'random'"
     ]
    }
   ],
   "source": [
    "from environment import environment_jax\n",
    "\n",
    "env = environment_jax.VehicleJobSchedulingEnvACE()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from environment import Environment\n",
    "\n",
    "env = Environment.VehicleJobSchedulingEnvACE()\n",
    "env.reset()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.parameters.cluster.machines[1].observe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "duration:  25  average_per_slot:  10\n",
      "Starting performance benchmark\n",
      "12679.175816391342 turns per second\n",
      "1056.5979846992784 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  25  average_per_slot:  30\n",
      "Starting performance benchmark\n",
      "12731.171350055083 turns per second\n",
      "1060.9309458379237 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  25  average_per_slot:  50\n",
      "Starting performance benchmark\n",
      "12692.710303681068 turns per second\n",
      "1057.725858640089 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  25  average_per_slot:  70\n",
      "Starting performance benchmark\n",
      "12447.760684349141 turns per second\n",
      "1037.3133903624284 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  25  average_per_slot:  90\n",
      "Starting performance benchmark\n",
      "12489.932789005727 turns per second\n",
      "1040.8277324171438 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  30  average_per_slot:  10\n",
      "Starting performance benchmark\n",
      "12766.805595751817 turns per second\n",
      "1063.9004663126514 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  30  average_per_slot:  30\n",
      "Starting performance benchmark\n",
      "12662.82735057095 turns per second\n",
      "1055.2356125475792 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  30  average_per_slot:  50\n",
      "Starting performance benchmark\n",
      "12595.536271335071 turns per second\n",
      "1049.628022611256 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  30  average_per_slot:  70\n",
      "Starting performance benchmark\n",
      "12495.82102308555 turns per second\n",
      "1041.3184185904624 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  30  average_per_slot:  90\n",
      "Starting performance benchmark\n",
      "12931.976991107913 turns per second\n",
      "1077.6647492589927 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  35  average_per_slot:  10\n",
      "Starting performance benchmark\n",
      "12884.796371381104 turns per second\n",
      "1073.7330309484253 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  35  average_per_slot:  30\n",
      "Starting performance benchmark\n",
      "12875.00360249619 turns per second\n",
      "1072.9169668746827 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  35  average_per_slot:  50\n",
      "Starting performance benchmark\n",
      "12687.333530403246 turns per second\n",
      "1057.2777942002706 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  35  average_per_slot:  70\n",
      "Starting performance benchmark\n",
      "12869.3117786244 turns per second\n",
      "1072.4426482187 cycles per second\n",
      "Finished performance benchmark\n",
      "duration:  35  average_per_slot:  90\n",
      "Starting performance benchmark\n",
      "12899.279734775095 turns per second\n",
      "1074.9399778979246 cycles per second\n",
      "Finished performance benchmark\n"
     ]
    }
   ],
   "source": [
    "import cProfile\n",
    "from pettingzoo.test import performance_benchmark\n",
    "from environment import Environment\n",
    "for i in range(25,40,5):\n",
    "    for j in range(10,100,20):\n",
    "        print(\"duration: \", i, \" average_per_slot: \", j)\n",
    "        para = Environment.VehicleJobSchedulingParameters(duration=i,average_per_slot=j)\n",
    "        env = Environment.VehicleJobSchedulingEnvACE()\n",
    "        performance_benchmark(env)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish Rate:  0.8738916256157635\n",
      "Finish Rate:  0.837245696400626\n",
      "Finish Rate:  0.7435723951285521\n",
      "Finish Rate:  0.6604767879548307\n",
      "Finish Rate:  0.5661592505854801\n",
      "Finish Rate:  0.5196389771017884\n",
      "Finish Rate:  0.4667235494880546\n",
      "Finish Rate:  0.43742098609355246\n",
      "Finish Rate:  0.3821138211382114\n"
     ]
    }
   ],
   "source": [
    "import cProfile\n",
    "from pettingzoo.test import performance_benchmark, api_test\n",
    "from environment import Environment\n",
    "\n",
    "for i in range(10,100,10):\n",
    "    para = Environment.VehicleJobSchedulingParameters(average_per_slot=i, duration=30)\n",
    "    env = Environment.VehicleJobSchedulingEnvACE(parameter=para)\n",
    "    env.reset()\n",
    "    for agent in env.agent_iter(100000):\n",
    "        env.step(env.action_space(agent).sample())   \n",
    "    print(\"Finish Rate: \",env.finished_job/env.total_job)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for agent in env.agent_iter(10):\n",
    "    action = 1\n",
    "    env.step(action)\n",
    "    print(action)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"env job finished rate\", env.finished_job/env.total_job)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from environment import Environment, AllocationMechanism, Machine\n",
    "\n",
    "env = Environment.VehicleJobSchedulingEnvACE()\n",
    "for job in env.get_job_next_step():\n",
    "    job1 = job\n",
    "    break\n",
    "cluster = env.parameters.cluster\n",
    "\n",
    "fp = AllocationMechanism.FirstPrice()\n",
    "bids = Machine.Bids(cluster,job1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bids.request_bids()\n",
    "bids.get_bids()\n",
    "bids.bids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tianshou.data import Collector\n",
    "from tianshou.env import DummyVectorEnv\n",
    "from tianshou.policy import MultiAgentPolicyManager, RandomPolicy\n",
    "\n",
    "import tianshou_setup\n",
    "\n",
    "env = tianshou_setup.get_env()\n",
    "policies = MultiAgentPolicyManager(\n",
    "    [\n",
    "        RandomPolicy(\n",
    "            observation_space=env.observation_space, action_space=env.action_space\n",
    "        )\n",
    "        for _ in range(len(env.agents))\n",
    "    ],\n",
    "    env,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pol = RandomPolicy(\n",
    "            observation_space=env.observation_space, action_space=env.action_space\n",
    "        )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pol.forward()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "from tianshou.data import Collector\n",
    "from tianshou.env import DummyVectorEnv, SubprocVectorEnv\n",
    "from tianshou.policy import MultiAgentPolicyManager\n",
    "\n",
    "import tianshou_setup\n",
    "from static_policy.TruthfulPolicy import TruthfulPolicy\n",
    "\n",
    "env = tianshou_setup.get_env_continous()\n",
    "\n",
    "\n",
    "def get_DummyVectorEnv_n(n):\n",
    "    return DummyVectorEnv([lambda: env for _ in range(n)])\n",
    "\n",
    "\n",
    "def get_SubprocVectorEnv_n(n):\n",
    "    return SubprocVectorEnv([lambda: env for _ in range(n)])\n",
    "\n",
    "\n",
    "def get_policy_manager():\n",
    "    env = tianshou_setup.get_env_continous()\n",
    "    policies = MultiAgentPolicyManager(\n",
    "        [\n",
    "            TruthfulPolicy(\n",
    "                observation_space=env.observation_space, action_space=env.action_space\n",
    "            )\n",
    "            for _ in range(len(env.agents))\n",
    "        ],\n",
    "        env,\n",
    "    )\n",
    "    return policies\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 10\n",
    "env = get_DummyVectorEnv_n(n)\n",
    "policies = get_policy_manager()\n",
    "collector = Collector(policies, env)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Execute one episode\n",
    "start = time.time()\n",
    "result = collector.collect(n_episode=10)\n",
    "end = time.time()\n",
    "print(f\"Time elapsed: {end-start}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp = policies.policies['Machine_0']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FinishRate(average_per_slot=10,machine_numbers=10):\n",
    "    from environment import Environment\n",
    "    para = Environment.VehicleJobSchedulingParameters(average_per_slot=average_per_slot, machine_numbers=machine_numbers)\n",
    "    env = Environment.VehicleJobSchedulingEnvACE(parameter=para)\n",
    "    env.reset()\n",
    "    for agent in env.agent_iter(100000):\n",
    "        env.step(env.action_space(agent).sample())\n",
    "    print(\"Finish Rate: \",env.finished_job/env.total_job)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average:  1\n",
      "Finish Rate:  0.8653846153846154\n",
      "average:  3\n",
      "Finish Rate:  0.9018181818181819\n",
      "average:  5\n",
      "Finish Rate:  0.8770161290322581\n",
      "average:  7\n",
      "Finish Rate:  0.8705547652916074\n",
      "average:  9\n",
      "Finish Rate:  0.84251968503937\n",
      "average:  11\n",
      "Finish Rate:  0.8443396226415094\n",
      "average:  13\n",
      "Finish Rate:  0.7830609212481426\n",
      "average:  15\n",
      "Finish Rate:  0.7724665391969407\n",
      "average:  17\n",
      "Finish Rate:  0.7380410022779044\n",
      "average:  19\n",
      "Finish Rate:  0.7107085775173149\n",
      "average:  21\n",
      "Finish Rate:  0.7013487475915221\n",
      "average:  23\n",
      "Finish Rate:  0.6526458616010855\n",
      "average:  25\n",
      "Finish Rate:  0.6032372680615871\n",
      "average:  27\n",
      "Finish Rate:  0.603460972017673\n",
      "average:  29\n",
      "Finish Rate:  0.5652474904811353\n",
      "average:  31\n",
      "Finish Rate:  0.5144059566202654\n",
      "average:  33\n",
      "Finish Rate:  0.5041398344066237\n",
      "average:  35\n",
      "Finish Rate:  0.48461755574372\n",
      "average:  37\n",
      "Finish Rate:  0.45884827767551933\n",
      "average:  39\n",
      "Finish Rate:  0.43522372528616027\n"
     ]
    }
   ],
   "source": [
    "for average in range(1,40,2):\n",
    "    print(\"average: \", average)\n",
    "    FinishRate(average_per_slot=average,machine_numbers=6)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average:  2\n",
      "Finish Rate:  0.9214659685863874\n",
      "average:  8\n",
      "Finish Rate:  0.8761467889908257\n",
      "average:  14\n",
      "Finish Rate:  0.8304446119065562\n",
      "average:  20\n",
      "Finish Rate:  0.8105749486652978\n",
      "average:  26\n",
      "Finish Rate:  0.7671130952380952\n",
      "average:  32\n",
      "Finish Rate:  0.7190490892250695\n",
      "average:  38\n",
      "Finish Rate:  0.685372340425532\n",
      "average:  44\n",
      "Finish Rate:  0.6315431679129844\n",
      "average:  50\n",
      "Finish Rate:  0.5862412761714856\n",
      "average:  56\n",
      "Finish Rate:  0.5447884227880564\n",
      "average:  62\n",
      "Finish Rate:  0.4992874109263658\n",
      "average:  68\n",
      "Finish Rate:  0.4751783590963139\n"
     ]
    }
   ],
   "source": [
    "for average in range(2,69,6):\n",
    "    print(\"average: \", average)\n",
    "    FinishRate(average_per_slot=average,machine_numbers=12)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average:  70\n",
      "Finish Rate:  0.6954293037163606\n",
      "average:  71\n",
      "Finish Rate:  0.6863938053097345\n",
      "average:  72\n",
      "Finish Rate:  0.6639825303671353\n",
      "average:  73\n",
      "Finish Rate:  0.6614386154678205\n",
      "average:  74\n",
      "Finish Rate:  0.6646570203644159\n",
      "average:  75\n",
      "Finish Rate:  0.6658644203770557\n"
     ]
    }
   ],
   "source": [
    "for average in range(70,76,1):\n",
    "    print(\"average: \", average)\n",
    "    FinishRate(average_per_slot=average,machine_numbers=24)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish Rate:  0.523389159310288\n"
     ]
    }
   ],
   "source": [
    "FinishRate(average_per_slot=120, machine_numbers=24)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "machine:  6  average:  10\n",
      "Finish Rate:  0.8121442125237192\n",
      "machine:  6  average:  20\n",
      "Finish Rate:  0.6793587174348698\n",
      "machine:  6  average:  30\n",
      "Finish Rate:  0.535857908847185\n",
      "machine:  6  average:  40\n",
      "Finish Rate:  0.45218476903870164\n",
      "machine:  6  average:  50\n",
      "Finish Rate:  0.36667995217218013\n",
      "machine:  6  average:  60\n",
      "Finish Rate:  0.30702179176755445\n",
      "machine:  6  average:  70\n",
      "Finish Rate:  0.27015032211882606\n",
      "machine:  6  average:  80\n",
      "Finish Rate:  0.2350432710397592\n",
      "machine:  6  average:  90\n",
      "Finish Rate:  0.21080903310677482\n",
      "machine:  12  average:  10\n",
      "Finish Rate:  0.8653100775193798\n",
      "machine:  12  average:  20\n",
      "Finish Rate:  0.8021148036253777\n",
      "machine:  12  average:  30\n",
      "Finish Rate:  0.7311827956989247\n",
      "machine:  12  average:  40\n",
      "Finish Rate:  0.6471898984897252\n",
      "machine:  12  average:  50\n",
      "Finish Rate:  0.5836909871244635\n",
      "machine:  12  average:  60\n",
      "Finish Rate:  0.5137344669718771\n",
      "machine:  12  average:  70\n",
      "Finish Rate:  0.46066282420749277\n",
      "machine:  12  average:  80\n",
      "Finish Rate:  0.40818597749474467\n",
      "machine:  12  average:  90\n",
      "Finish Rate:  0.3758778285586891\n",
      "machine:  24  average:  10\n",
      "Finish Rate:  0.8796592119275826\n",
      "machine:  24  average:  20\n",
      "Finish Rate:  0.869281045751634\n",
      "machine:  24  average:  30\n",
      "Finish Rate:  0.8613569321533924\n",
      "machine:  24  average:  40\n",
      "Finish Rate:  0.8319899244332494\n",
      "machine:  24  average:  50\n",
      "Finish Rate:  0.7788344799676244\n",
      "machine:  24  average:  60\n",
      "Finish Rate:  0.7243116185359302\n",
      "machine:  24  average:  70\n",
      "Finish Rate:  0.6901766784452297\n",
      "machine:  24  average:  80\n",
      "Finish Rate:  0.66278049392002\n",
      "machine:  24  average:  90\n",
      "Finish Rate:  0.6040080609046127\n"
     ]
    }
   ],
   "source": [
    "for machine in [6,12,24]:\n",
    "    for average in range(10,100,10):\n",
    "        print(\"machine: \", machine, \" average: \", average)\n",
    "        FinishRate(average,machine)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-30 11:12:30,610\tINFO worker.py:1627 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8266 \u001b[39m\u001b[22m\n",
      "2023-09-30 11:12:31,803\tWARNING deprecation.py:50 -- DeprecationWarning: `rllib/algorithms/a3c/a3c.py` has been deprecated. Use `rllib_contrib/a3c/` instead. This will raise an error in the future!\n",
      "2023-09-30 11:12:31,804\tWARNING deprecation.py:50 -- DeprecationWarning: `rllib/algorithms/a3c/a3c.py` has been deprecated. Use `rllib_contrib/a3c/` instead. This will raise an error in the future!\n",
      "2023-09-30 11:12:31,870\tWARNING deprecation.py:50 -- DeprecationWarning: `rllib/algorithms/maml/maml.py` has been deprecated. Use `rllib_contrib/maml/` instead. This will raise an error in the future!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'extra_python_environs_for_driver': {}, 'extra_python_environs_for_worker': {}, 'num_gpus': 1, 'num_cpus_per_worker': 1, 'num_gpus_per_worker': 0, '_fake_gpus': False, 'num_learner_workers': 0, 'num_gpus_per_learner_worker': 0, 'num_cpus_per_learner_worker': 1, 'local_gpu_idx': 0, 'custom_resources_per_worker': {}, 'placement_strategy': 'PACK', 'eager_tracing': False, 'eager_max_retraces': 20, 'tf_session_args': {'intra_op_parallelism_threads': 2, 'inter_op_parallelism_threads': 2, 'gpu_options': {'allow_growth': True}, 'log_device_placement': False, 'device_count': {'CPU': 1}, 'allow_soft_placement': True}, 'local_tf_session_args': {'intra_op_parallelism_threads': 8, 'inter_op_parallelism_threads': 8}, 'env': 'VJS_6_30', 'env_config': {}, 'observation_space': None, 'action_space': None, 'env_task_fn': None, 'render_env': False, 'clip_rewards': None, 'normalize_actions': True, 'clip_actions': False, 'disable_env_checking': True, 'is_atari': None, 'auto_wrap_old_gym_envs': True, 'num_envs_per_worker': 1, 'sample_collector': <class 'ray.rllib.evaluation.collectors.simple_list_collector.SimpleListCollector'>, 'sample_async': False, 'enable_connectors': True, 'rollout_fragment_length': 30, 'batch_mode': 'complete_episodes', 'remote_worker_envs': False, 'remote_env_batch_wait_ms': 0, 'validate_workers_after_construction': True, 'preprocessor_pref': 'deepmind', 'observation_filter': 'NoFilter', 'synchronize_filters': True, 'compress_observations': False, 'enable_tf1_exec_eagerly': False, 'sampler_perf_stats_ema_coef': None, 'gamma': 0.99, 'lr': 0.01, 'lr_schedule': None, 'grad_clip': None, 'grad_clip_by': 'global_norm', 'train_batch_size': 256, 'model': {'_disable_preprocessor_api': False, '_disable_action_flattening': False, 'fcnet_hiddens': [256, 256], 'fcnet_activation': 'tanh', 'conv_filters': None, 'conv_activation': 'relu', 'post_fcnet_hiddens': [], 'post_fcnet_activation': 'relu', 'free_log_std': False, 'no_final_linear': False, 'vf_share_layers': True, 'use_lstm': False, 'max_seq_len': 20, 'lstm_cell_size': 256, 'lstm_use_prev_action': False, 'lstm_use_prev_reward': False, '_time_major': False, 'use_attention': False, 'attention_num_transformer_units': 1, 'attention_dim': 64, 'attention_num_heads': 1, 'attention_head_dim': 32, 'attention_memory_inference': 50, 'attention_memory_training': 50, 'attention_position_wise_mlp_dim': 32, 'attention_init_gru_gate_bias': 2.0, 'attention_use_n_prev_actions': 0, 'attention_use_n_prev_rewards': 0, 'framestack': True, 'dim': 84, 'grayscale': False, 'zero_mean': True, 'custom_model': None, 'custom_model_config': {}, 'custom_action_dist': None, 'custom_preprocessor': None, 'encoder_latent_dim': None, 'always_check_shapes': False, 'lstm_use_prev_action_reward': -1, '_use_default_native_models': -1}, 'optimizer': {}, 'max_requests_in_flight_per_sampler_worker': 2, '_learner_class': None, '_enable_learner_api': False, 'explore': True, 'exploration_config': {'type': 'StochasticSampling'}, 'policy_states_are_swappable': False, 'input_config': {}, 'actions_in_input_normalized': False, 'postprocess_inputs': False, 'shuffle_buffer_size': 0, 'output': None, 'output_config': {}, 'output_compress_columns': ['obs', 'new_obs'], 'output_max_file_size': 67108864, 'offline_sampling': False, 'evaluation_interval': None, 'evaluation_duration': 10, 'evaluation_duration_unit': 'episodes', 'evaluation_sample_timeout_s': 180.0, 'evaluation_parallel_to_training': False, 'evaluation_config': None, 'off_policy_estimation_methods': {}, 'ope_split_batch_by_episode': True, 'evaluation_num_workers': 0, 'always_attach_evaluation_results': False, 'enable_async_evaluation': False, 'in_evaluation': False, 'sync_filters_on_rollout_workers_timeout_s': 60.0, 'keep_per_episode_custom_metrics': False, 'metrics_episode_collection_timeout_s': 60.0, 'metrics_num_episodes_for_smoothing': 100, 'min_time_s_per_iteration': 1, 'min_train_timesteps_per_iteration': 0, 'min_sample_timesteps_per_iteration': 100, 'export_native_model_files': False, 'checkpoint_trainable_policies_only': False, 'logger_creator': None, 'logger_config': None, 'log_level': 'WARN', 'log_sys_usage': True, 'fake_sampler': False, 'seed': None, 'worker_cls': None, 'ignore_worker_failures': False, 'recreate_failed_workers': False, 'max_num_worker_restarts': 1000, 'delay_between_worker_restarts_s': 60.0, 'restart_failed_sub_environments': False, 'num_consecutive_worker_failures_tolerance': 100, 'worker_health_probe_timeout_s': 60, 'worker_restore_timeout_s': 1800, 'rl_module_spec': None, '_enable_rl_module_api': False, '_AlgorithmConfig__prior_exploration_config': None, '_tf_policy_handles_more_than_one_loss': False, '_disable_preprocessor_api': False, '_disable_action_flattening': False, '_disable_execution_plan_api': True, '_disable_initialize_loss_from_dummy_batch': False, 'simple_optimizer': -1, 'replay_sequence_length': None, 'horizon': -1, 'soft_horizon': -1, 'no_done_at_end': -1, 'twin_q': True, 'q_model_config': {'fcnet_hiddens': [256, 256], 'fcnet_activation': 'relu', 'post_fcnet_hiddens': [], 'post_fcnet_activation': None, 'custom_model': None, 'custom_model_config': {}}, 'policy_model_config': {'fcnet_hiddens': [256, 256], 'fcnet_activation': 'relu', 'post_fcnet_hiddens': [], 'post_fcnet_activation': None, 'custom_model': None, 'custom_model_config': {}}, 'tau': 0.005, 'initial_alpha': 1.0, 'target_entropy': 'auto', 'n_step': 1, 'replay_buffer_config': {'_enable_replay_buffer_api': True, 'type': 'MultiAgentPrioritizedReplayBuffer', 'capacity': 1000000, 'prioritized_replay': False, 'prioritized_replay_alpha': 0.6, 'prioritized_replay_beta': 0.4, 'prioritized_replay_eps': 1e-06, 'worker_side_prioritization': False}, 'store_buffer_in_checkpoints': False, 'training_intensity': None, 'optimization': {'actor_learning_rate': 0.0003, 'critic_learning_rate': 0.0003, 'entropy_learning_rate': 0.0003}, 'target_network_update_freq': 0, 'num_steps_sampled_before_learning_starts': 1500, '_deterministic_loss': False, '_use_beta_distribution': False, 'use_state_preprocessor': -1, 'worker_side_prioritization': -1, 'input': 'sampler', 'multiagent': {'policies': {'Machine_0': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {}), 'Machine_1': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {}), 'Machine_2': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {}), 'Machine_3': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {}), 'Machine_4': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {}), 'Machine_5': (None, Box(0.0, [4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03\n",
      " 4.097e+03 4.097e+03 4.097e+03 4.097e+03 4.097e+03 1.000e+02 1.000e+02\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00\n",
      " 1.000e+00 1.000e+00 1.000e+00], (73,), float64), Box(0.5, 2.0, (1,), float32), {})}, 'policy_mapping_fn': <function train_sac.<locals>.<lambda> at 0x7f87ea870550>, 'policies_to_train': None, 'policy_map_capacity': 100, 'policy_map_cache': -1, 'count_steps_by': 'env_steps', 'observation_fn': None}, 'callbacks': <class 'ray.rllib.algorithms.callbacks.DefaultCallbacks'>, 'create_env_on_driver': False, 'custom_eval_function': None, 'framework': 'torch', 'num_cpus_for_driver': 1, 'num_workers': 10}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-09-30 11:13:32</td></tr>\n",
       "<tr><td>Running for: </td><td>00:01:00.12        </td></tr>\n",
       "<tr><td>Memory:      </td><td>43.3/251.8 GiB     </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using FIFO scheduling algorithm.<br>Logical resource usage: 11.0/56 CPUs, 1.0/2 GPUs (0.0/1.0 accelerator_type:G)\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name              </th><th>status  </th><th>loc               </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    ts</th><th style=\"text-align: right;\">     reward</th><th style=\"text-align: right;\">  episode_reward_max</th><th style=\"text-align: right;\">  episode_reward_min</th><th style=\"text-align: right;\">  episode_len_mean</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>SAC_VJS_6_30_31da1_00000</td><td>RUNNING </td><td>192.168.3.6:818076</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         42.4586</td><td style=\"text-align: right;\">110755</td><td style=\"text-align: right;\">3.91781e+07</td><td style=\"text-align: right;\">         4.45646e+07</td><td style=\"text-align: right;\">          3.6943e+07</td><td style=\"text-align: right;\">           11075.5</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(SAC pid=818076)\u001b[0m 2023-09-30 11:12:38,962\tWARNING algorithm_config.py:643 -- Cannot create SACConfig from given `config_dict`! Property __stdout_file__ not supported.\n",
      "\u001b[2m\u001b[36m(SAC pid=818076)\u001b[0m 2023-09-30 11:12:38,979\tINFO algorithm.py:536 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
      "\u001b[2m\u001b[36m(SAC pid=818076)\u001b[0m 2023-09-30 11:12:49,208\tINFO trainable.py:173 -- Trainable.setup took 10.231 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m 2023-09-30 11:13:10,966\tWARNING env_runner_v2.py:154 -- More than 11180 observations in 11180 env steps for episode 814477653228105328 are buffered in the sampler. If this is more than you expected, check that that you set a horizon on your environment correctly and that it terminates at some point. Note: In multi-agent environments, `rollout_fragment_length` sets the batch size based on (across-agents) environment steps, not the steps of individual agents, which can result in unexpectedly large batches.Also, you may be waiting for your Env to terminate (batch_mode=`complete_episodes`). Make sure it does at some point.\n",
      "\u001b[2m\u001b[36m(SAC pid=818076)\u001b[0m 2023-09-30 11:13:13,132\tWARNING multi_agent_prioritized_replay_buffer.py:215 -- Adding batches with column `weights` to this buffer while providing weights as a call argument to the add method results in the column being overwritten.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name              </th><th style=\"text-align: right;\">  agent_timesteps_total</th><th>connector_metrics                                                                                                                                            </th><th>counters                                                                                                                                                                                    </th><th>custom_metrics  </th><th>date               </th><th>done  </th><th style=\"text-align: right;\">  episode_len_mean</th><th>episode_media  </th><th style=\"text-align: right;\">  episode_reward_max</th><th style=\"text-align: right;\">  episode_reward_mean</th><th style=\"text-align: right;\">  episode_reward_min</th><th style=\"text-align: right;\">  episodes_this_iter</th><th style=\"text-align: right;\">  episodes_total</th><th>hostname        </th><th>info  </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip    </th><th style=\"text-align: right;\">  num_agent_steps_sampled</th><th style=\"text-align: right;\">  num_agent_steps_trained</th><th style=\"text-align: right;\">  num_env_steps_sampled</th><th style=\"text-align: right;\">  num_env_steps_sampled_this_iter</th><th style=\"text-align: right;\">  num_env_steps_sampled_throughput_per_sec</th><th style=\"text-align: right;\">  num_env_steps_trained</th><th style=\"text-align: right;\">  num_env_steps_trained_this_iter</th><th style=\"text-align: right;\">  num_env_steps_trained_throughput_per_sec</th><th style=\"text-align: right;\">  num_faulty_episodes</th><th style=\"text-align: right;\">  num_healthy_workers</th><th style=\"text-align: right;\">  num_in_flight_async_reqs</th><th style=\"text-align: right;\">  num_remote_worker_restarts</th><th style=\"text-align: right;\">  num_steps_trained_this_iter</th><th>perf                                                                                                                                                                                                                                                         </th><th style=\"text-align: right;\">   pid</th><th>policy_reward_max                                                                                                                                                                              </th><th>policy_reward_mean                                                                                                                                                                               </th><th>policy_reward_min                                                                                                                                                                                 </th><th>sampler_perf                                                                                                                                                                                                    </th><th>sampler_results                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              </th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th>timers                                                                                                                                                                                                             </th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_total</th><th style=\"text-align: right;\">  training_iteration</th><th>trial_id   </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>SAC_VJS_6_30_31da1_00000</td><td style=\"text-align: right;\">                 110755</td><td>{&#x27;ObsPreprocessorConnector_ms&#x27;: 0.005630652109781901, &#x27;StateBufferConnector_ms&#x27;: 0.0044707457224528, &#x27;ViewRequirementAgentConnector_ms&#x27;: 0.12716094652811685}</td><td>{&#x27;num_env_steps_sampled&#x27;: 110755, &#x27;num_env_steps_trained&#x27;: 258, &#x27;num_agent_steps_sampled&#x27;: 110755, &#x27;num_agent_steps_trained&#x27;: 258, &#x27;last_target_update_ts&#x27;: 110755, &#x27;num_target_updates&#x27;: 1}</td><td>{}              </td><td>2023-09-30_11-13-31</td><td>False </td><td style=\"text-align: right;\">           11075.5</td><td>{}             </td><td style=\"text-align: right;\">         4.45646e+07</td><td style=\"text-align: right;\">          3.91781e+07</td><td style=\"text-align: right;\">          3.6943e+07</td><td style=\"text-align: right;\">                  10</td><td style=\"text-align: right;\">              10</td><td>lwh-Super-Server</td><td>{&#x27;learner&#x27;: {&#x27;Machine_0&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: 0.6877908706665039, &#x27;critic_loss&#x27;: 5141.9453125, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 0.99970007, &#x27;log_alpha_value&#x27;: -0.00029999993, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: 0.760205090045929, &#x27;mean_q&#x27;: -0.07945867627859116, &#x27;max_q&#x27;: 0.14960968494415283, &#x27;min_q&#x27;: -0.28345489501953125}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([4.0665527e+04, 5.4440305e-02, 1.1243813e+00, 4.8830611e-01,\n",
       "       1.2530667e+00, 8.2524086e+04, 1.6063209e+00, 1.3335884e+00,\n",
       "       2.1081969e-02, 1.2101886e+00, 1.6074028e+00, 1.6620512e+00,\n",
       "       2.8746176e-01, 9.6830225e-01, 1.3710349e+00, 1.3568031e+00,\n",
       "       3.3793747e-01, 1.1528366e+00, 3.2543977e+04, 1.5321329e+00,\n",
       "       7.0614624e-01, 4.7325560e-01, 1.6111774e+00, 7.7656716e-02,\n",
       "       9.8603183e-01, 1.4736912e+00, 6.5347090e+04, 1.3899674e+00,\n",
       "       1.4403410e+00, 1.2815191e+00, 1.1749709e-01, 1.5255405e+00,\n",
       "       7.3588169e-01, 1.5035071e+00, 2.0056548e+00, 1.6878421e+00,\n",
       "       1.7585278e+00, 6.7455769e-01, 3.8693756e-01, 7.8697848e-01,\n",
       "       2.9450893e-01, 2.4779024e+00, 4.0819579e-01], dtype=float32), &#x27;mean_td_error&#x27;: 5142.36865234375}, &#x27;Machine_1&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: 8.174642562866211, &#x27;critic_loss&#x27;: 5674.10498046875, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 1.0003, &#x27;log_alpha_value&#x27;: 0.00029999996, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: -0.05043136700987816, &#x27;mean_q&#x27;: 5.080766677856445, &#x27;max_q&#x27;: 5.831745147705078, &#x27;min_q&#x27;: 3.950099229812622}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([1.7305450e+05, 5.4912758e+00, 1.2189648e+01, 9.5074043e+00,\n",
       "       1.2270411e+01, 1.0431404e+01, 7.0626703e+04, 4.0478263e+00,\n",
       "       5.1994123e+00, 3.8193402e+00, 1.2340724e+01, 6.5207343e+00,\n",
       "       9.0678062e+00, 8.2051783e+00, 4.2511396e+00, 9.2943678e+00,\n",
       "       1.2387680e+01, 3.9857445e+00, 9.8472404e+00, 3.8245883e+00,\n",
       "       1.2343034e+01, 4.1130624e+00, 1.0952567e+01, 1.1791536e+01,\n",
       "       4.2248945e+00, 9.5534048e+00, 1.2315926e+01, 3.4921072e+00,\n",
       "       1.2722763e+01, 1.2999750e+01, 3.5853291e+00, 4.1947675e+00,\n",
       "       5.3616371e+00, 1.0939082e+01, 3.1626801e+00, 3.6698041e+00,\n",
       "       1.2001844e+01, 1.0544850e+01, 3.7296329e+00, 3.9835219e+00,\n",
       "       1.2194521e+01, 3.7314506e+00, 1.1973492e+01], dtype=float32), &#x27;mean_td_error&#x27;: 5674.591796875}, &#x27;Machine_2&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: -0.07856561243534088, &#x27;critic_loss&#x27;: 2327.988525390625, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 0.99970007, &#x27;log_alpha_value&#x27;: -0.00029999996, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: 0.23459558188915253, &#x27;mean_q&#x27;: 0.02755546011030674, &#x27;max_q&#x27;: 0.7291316986083984, &#x27;min_q&#x27;: -0.7601333856582642}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([5.8758605e-01, 8.0582562e+04, 4.1056852e+00, 1.9486709e+04,\n",
       "       3.5430676e-01, 3.3099914e+00, 1.3380589e+00, 6.8220079e-01,\n",
       "       2.3633614e+00, 9.1002727e-01, 1.6930327e-01, 4.0229198e-01,\n",
       "       8.2213491e-01, 3.5676429e-01, 3.2816434e-01, 6.7999506e-01,\n",
       "       2.5508823e+00, 1.2836016e+00, 3.3226304e+00, 8.2756311e-01,\n",
       "       6.8102640e-01, 2.8707761e-01, 1.7927976e+00, 8.5441250e-01,\n",
       "       5.3936410e-01, 1.8339012e+00, 3.1306183e-01, 6.4521837e-01,\n",
       "       1.5526724e+00, 4.1644615e-01, 1.7772462e+00, 2.1593404e+00,\n",
       "       9.9099046e-01, 2.9086629e-01, 3.8315945e+00, 4.5075154e-01,\n",
       "       3.4123871e+00, 1.3278430e+00, 1.0399585e+00, 6.2660277e-01,\n",
       "       1.4658800e+00, 7.8512627e-01, 2.6628187e-01], dtype=float32), &#x27;mean_td_error&#x27;: 2328.3955078125}, &#x27;Machine_3&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: 0.26429811120033264, &#x27;critic_loss&#x27;: 2172.265869140625, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 0.99970007, &#x27;log_alpha_value&#x27;: -0.00029999996, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: 0.35043472051620483, &#x27;mean_q&#x27;: -0.23442591726779938, &#x27;max_q&#x27;: -0.04695901274681091, &#x27;min_q&#x27;: -0.44547539949417114}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([1.7923430e+00, 5.7504106e-01, 1.9454247e-01, 6.2672466e-01,\n",
       "       5.2029550e-01, 7.1262741e-01, 1.6206093e-01, 1.3882105e+00,\n",
       "       5.7498455e-01, 6.1877477e-01, 8.4421903e-01, 6.5886045e-01,\n",
       "       7.6197994e-01, 6.5907938e+04, 4.2457312e-01, 1.7565420e+00,\n",
       "       2.9460657e-01, 7.2171390e-01, 7.5608397e-01, 1.3203483e+00,\n",
       "       3.1762588e-01, 5.5810446e-01, 2.7485277e+04, 1.9721593e-01,\n",
       "       6.4488834e-01, 7.1502197e-01, 4.9310675e-01, 4.4309089e-01,\n",
       "       2.9362506e-01, 3.3387545e-01, 4.9404278e-01, 3.1843308e-01,\n",
       "       1.1998708e+00, 2.2215376e+00, 6.3516480e-01, 3.7205288e-01,\n",
       "       1.0699122e+00, 2.4758452e-01, 6.7817312e-01, 5.6747800e-01,\n",
       "       2.1458712e+00, 1.2268114e+00, 7.3482060e-01], dtype=float32), &#x27;mean_td_error&#x27;: 2172.647216796875}, &#x27;Machine_4&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: 15.428377151489258, &#x27;critic_loss&#x27;: 2081.624755859375, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 1.0003, &#x27;log_alpha_value&#x27;: 0.0003, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: 0.9788821935653687, &#x27;mean_q&#x27;: 6.983963489532471, &#x27;max_q&#x27;: 7.0963521003723145, &#x27;min_q&#x27;: 6.492913246154785}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([1.06187582e+01, 1.04036846e+01, 1.11610632e+01, 1.08321686e+01,\n",
       "       1.05897369e+01, 1.13831577e+01, 1.03485432e+01, 9.34339046e+00,\n",
       "       1.00174732e+01, 3.58909619e+03, 1.06706810e+01, 1.02796345e+01,\n",
       "       1.03148298e+01, 9.64211845e+00, 1.11754723e+01, 1.09429226e+01,\n",
       "       9.17383194e+00, 1.14269009e+01, 1.13760605e+01, 1.00352516e+01,\n",
       "       1.14164133e+01, 1.14908838e+01, 1.11179123e+01, 1.12190189e+01,\n",
       "       1.11555882e+01, 7.04760000e+04, 1.13282681e+01, 1.13002310e+01,\n",
       "       1.12677288e+01, 1.10680885e+01, 9.39256859e+00, 1.06918478e+01,\n",
       "       1.07006235e+01, 1.07546120e+01, 1.04136829e+01, 1.50342031e+04,\n",
       "       1.11747332e+01, 1.14203873e+01, 1.11885738e+01, 1.15197659e+01,\n",
       "       1.13213043e+01, 1.10898743e+01, 1.12960320e+01], dtype=float32), &#x27;mean_td_error&#x27;: 2082.12451171875}, &#x27;Machine_5&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;actor_loss&#x27;: 1.1621485948562622, &#x27;critic_loss&#x27;: 6825.7041015625, &#x27;alpha_loss&#x27;: 0.0, &#x27;alpha_value&#x27;: 0.99970007, &#x27;log_alpha_value&#x27;: -0.00029999996, &#x27;target_entropy&#x27;: -1.0, &#x27;policy_t&#x27;: -0.027112172916531563, &#x27;mean_q&#x27;: -1.792523741722107, &#x27;max_q&#x27;: -1.2957220077514648, &#x27;min_q&#x27;: -2.3339755535125732}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1.0, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 0.0, &#x27;td_error&#x27;: array([1.23245016e+05, 1.89375520e+00, 1.45384383e+00, 1.37705553e+00,\n",
       "       1.11858439e+00, 1.69661248e+00, 1.64045048e+00, 1.67600369e+00,\n",
       "       1.85766935e+00, 1.73400164e+00, 1.70464540e+00, 1.89115191e+00,\n",
       "       1.81754947e+00, 1.36852145e+00, 8.21454688e+04, 1.05796814e+00,\n",
       "       1.30691934e+00, 1.21382844e+00, 1.90995109e+00, 1.37077820e+00,\n",
       "       2.02329254e+00, 1.05481291e+00, 1.51228583e+00, 1.64141321e+00,\n",
       "       1.07416439e+00, 1.87606573e+00, 8.80733125e+04, 2.01847625e+00,\n",
       "       1.93169868e+00, 1.66582131e+00, 1.47454548e+00, 1.52608204e+00,\n",
       "       1.14475298e+00, 2.00179100e+00, 2.07290244e+00, 1.32667160e+00,\n",
       "       2.01289749e+00, 1.34144616e+00, 1.17963648e+00, 1.03355098e+00,\n",
       "       1.08432078e+00, 1.45389867e+00, 1.27924371e+00], dtype=float32), &#x27;mean_td_error&#x27;: 6826.17724609375}}, &#x27;num_env_steps_sampled&#x27;: 110755, &#x27;num_env_steps_trained&#x27;: 258, &#x27;num_agent_steps_sampled&#x27;: 110755, &#x27;num_agent_steps_trained&#x27;: 258, &#x27;last_target_update_ts&#x27;: 110755, &#x27;num_target_updates&#x27;: 1}       </td><td style=\"text-align: right;\">                         1</td><td>192.168.3.6</td><td style=\"text-align: right;\">                   110755</td><td style=\"text-align: right;\">                      258</td><td style=\"text-align: right;\">                 110755</td><td style=\"text-align: right;\">                           110755</td><td style=\"text-align: right;\">                                   2610.32</td><td style=\"text-align: right;\">                    258</td><td style=\"text-align: right;\">                              258</td><td style=\"text-align: right;\">                                   6.08065</td><td style=\"text-align: right;\">                    0</td><td style=\"text-align: right;\">                   10</td><td style=\"text-align: right;\">                         0</td><td style=\"text-align: right;\">                           0</td><td style=\"text-align: right;\">                          258</td><td>{&#x27;cpu_util_percent&#x27;: 18.28444444444444, &#x27;ram_util_percent&#x27;: 17.008888888888894, &#x27;gpu_util_percent0&#x27;: 0.5993333333333333, &#x27;vram_util_percent0&#x27;: 0.8309172453703706, &#x27;gpu_util_percent1&#x27;: 0.00044444444444444447, &#x27;vram_util_percent1&#x27;: 0.00032552083333333337}</td><td style=\"text-align: right;\">818076</td><td>{&#x27;Machine_2&#x27;: 9906780.447603539, &#x27;Machine_5&#x27;: 9387044.388747254, &#x27;Machine_4&#x27;: 5783864.870646471, &#x27;Machine_3&#x27;: 4257275.229611814, &#x27;Machine_0&#x27;: 6479269.19855386, &#x27;Machine_1&#x27;: 10473960.10680602}</td><td>{&#x27;Machine_2&#x27;: 8781241.356379366, &#x27;Machine_5&#x27;: 8778715.720527632, &#x27;Machine_4&#x27;: 3571719.427174209, &#x27;Machine_3&#x27;: 3603660.816933346, &#x27;Machine_0&#x27;: 5007179.5825243415, &#x27;Machine_1&#x27;: 9435590.829809576}</td><td>{&#x27;Machine_2&#x27;: 7798972.249257327, &#x27;Machine_5&#x27;: 7372601.303536683, &#x27;Machine_4&#x27;: 1725123.1581086365, &#x27;Machine_3&#x27;: 2638908.0997323096, &#x27;Machine_0&#x27;: 4304654.594082534, &#x27;Machine_1&#x27;: 8631553.000595061}</td><td>{&#x27;mean_raw_obs_processing_ms&#x27;: 0.3016310477046461, &#x27;mean_inference_ms&#x27;: 1.250758361285411, &#x27;mean_action_processing_ms&#x27;: 0.20352159306542755, &#x27;mean_env_wait_ms&#x27;: 0.12070695063656237, &#x27;mean_env_render_ms&#x27;: 0.0}</td><td>{&#x27;episode_reward_max&#x27;: 44564626.56391185, &#x27;episode_reward_min&#x27;: 36942991.72150791, &#x27;episode_reward_mean&#x27;: 39178107.733348474, &#x27;episode_len_mean&#x27;: 11075.5, &#x27;episode_media&#x27;: {}, &#x27;episodes_this_iter&#x27;: 10, &#x27;policy_reward_min&#x27;: {&#x27;Machine_2&#x27;: 7798972.249257327, &#x27;Machine_5&#x27;: 7372601.303536683, &#x27;Machine_4&#x27;: 1725123.1581086365, &#x27;Machine_3&#x27;: 2638908.0997323096, &#x27;Machine_0&#x27;: 4304654.594082534, &#x27;Machine_1&#x27;: 8631553.000595061}, &#x27;policy_reward_max&#x27;: {&#x27;Machine_2&#x27;: 9906780.447603539, &#x27;Machine_5&#x27;: 9387044.388747254, &#x27;Machine_4&#x27;: 5783864.870646471, &#x27;Machine_3&#x27;: 4257275.229611814, &#x27;Machine_0&#x27;: 6479269.19855386, &#x27;Machine_1&#x27;: 10473960.10680602}, &#x27;policy_reward_mean&#x27;: {&#x27;Machine_2&#x27;: 8781241.356379366, &#x27;Machine_5&#x27;: 8778715.720527632, &#x27;Machine_4&#x27;: 3571719.427174209, &#x27;Machine_3&#x27;: 3603660.816933346, &#x27;Machine_0&#x27;: 5007179.5825243415, &#x27;Machine_1&#x27;: 9435590.829809576}, &#x27;custom_metrics&#x27;: {}, &#x27;hist_stats&#x27;: {&#x27;episode_reward&#x27;: [40930859.60915342, 39703735.29956111, 37442999.90479669, 37302525.277771175, 38430229.64975363, 41841952.29985553, 37406038.56926617, 44564626.56391185, 36942991.72150791, 37215118.4379073], &#x27;episode_lengths&#x27;: [11180, 11144, 11037, 10723, 11293, 10861, 11090, 11322, 10971, 11134], &#x27;policy_Machine_2_reward&#x27;: [9056348.17918519, 9771461.09495578, 7798972.249257327, 8726141.921534631, 8426430.359457463, 9311726.048077293, 7974784.807999194, 9906780.447603539, 8285436.02737236, 8554332.428350898], &#x27;policy_Machine_5_reward&#x27;: [9224877.104262536, 7372601.303536683, 8413679.63646838, 8468394.90954867, 8586144.418477328, 9387044.388747254, 8980337.414001409, 8991111.905710343, 9157631.267742252, 9205334.856781462], &#x27;policy_Machine_4_reward&#x27;: [2873377.8017750904, 5441767.677134634, 2507373.8305901196, 3297191.9611710208, 4429675.712143716, 3996167.6708685714, 3050046.116148341, 5783864.870646471, 2612605.4731554906, 1725123.1581086365], &#x27;policy_Machine_3_reward&#x27;: [4116593.231486857, 3626056.774901122, 4041732.521159202, 3648031.5458087325, 2638908.0997323096, 3294535.898992002, 4040316.5909834504, 4257275.229611814, 2755603.725015521, 3617554.5516424477], &#x27;policy_Machine_0_reward&#x27;: [5185703.1856377125, 4552182.874254763, 4304654.594082534, 4489938.501571476, 4738112.4010511935, 6479269.19855386, 4729000.639538705, 5922831.132450521, 4658215.6642844975, 5011887.63381815], &#x27;policy_Machine_1_reward&#x27;: [10473960.10680602, 8939665.574778136, 10376587.073239097, 8672826.438136635, 9610958.658891607, 9373209.094616553, 8631553.000595061, 9702762.977889169, 9473499.563937776, 9100885.809205705]}, &#x27;sampler_perf&#x27;: {&#x27;mean_raw_obs_processing_ms&#x27;: 0.3016310477046461, &#x27;mean_inference_ms&#x27;: 1.250758361285411, &#x27;mean_action_processing_ms&#x27;: 0.20352159306542755, &#x27;mean_env_wait_ms&#x27;: 0.12070695063656237, &#x27;mean_env_render_ms&#x27;: 0.0}, &#x27;num_faulty_episodes&#x27;: 0, &#x27;connector_metrics&#x27;: {&#x27;ObsPreprocessorConnector_ms&#x27;: 0.005630652109781901, &#x27;StateBufferConnector_ms&#x27;: 0.0044707457224528, &#x27;ViewRequirementAgentConnector_ms&#x27;: 0.12716094652811685}}</td><td style=\"text-align: right;\">             42.4586</td><td style=\"text-align: right;\">           42.4586</td><td style=\"text-align: right;\">       42.4586</td><td>{&#x27;training_iteration_time_ms&#x27;: 42429.443, &#x27;sample_time_ms&#x27;: 21824.654, &#x27;load_time_ms&#x27;: 47.391, &#x27;load_throughput&#x27;: 5444.024, &#x27;learn_time_ms&#x27;: 341.266, &#x27;learn_throughput&#x27;: 756.009, &#x27;synch_weights_time_ms&#x27;: 67.445}</td><td style=\"text-align: right;\"> 1696043611</td><td style=\"text-align: right;\">           110755</td><td style=\"text-align: right;\">                   1</td><td>31da1_00000</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-30 11:13:31,857\tWARNING tune.py:192 -- Stop signal received (e.g. via SIGINT/Ctrl+C), ending Ray Tune run. This will try to checkpoint the experiment state one last time. Press CTRL+C (or send SIGINT/SIGKILL/SIGTERM) to skip. \n",
      "2023-09-30 11:13:42,053\tINFO tune.py:1111 -- Total run time: 70.25 seconds (60.03 seconds for the tuning loop).\n",
      "2023-09-30 11:13:42,054\tWARNING tune.py:1126 -- Experiment has been interrupted, but the most recent state was saved.\n",
      "Resume experiment with: tune.run(..., resume=True)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1438, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1378, in ray._raylet.execute_task.function_executor\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/_private/function_manager.py\", line 724, in actor_method_executor\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     return method(__ray_actor, *args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/util/tracing/tracing_helper.py\", line 464, in _resume_span\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     return method(self, *_args, **_kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/actor_manager.py\", line 176, in apply\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     return func(self, *args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/execution/rollout_ops.py\", line 86, in <lambda>\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     lambda w: w.sample(), local_worker=False, healthy_only=True\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/util/tracing/tracing_helper.py\", line 464, in _resume_span\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     return method(self, *_args, **_kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 915, in sample\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     batches = [self.input_reader.next()]\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/sampler.py\", line 92, in next\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     batches = [self.get_data()]\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/sampler.py\", line 277, in get_data\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     item = next(self._env_runner)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/env_runner_v2.py\", line 323, in run\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     outputs = self.step()\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/env_runner_v2.py\", line 368, in step\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     ] = self._process_policy_eval_results(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/env_runner_v2.py\", line 1128, in _process_policy_eval_results\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     action_to_send, rnn_states, fetches = policy.action_connectors(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/connectors/action/pipeline.py\", line 30, in __call__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     ac_data = c(ac_data)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/util/timer.py\", line 30, in __exit__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     self.push(time_delta)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/util/timer.py\", line 37, in push\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     self.count += 1\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/_private/worker.py\", line 858, in sigterm_handler\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     sys.exit(1)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m SystemExit: 1\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1431, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1513, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/_private/worker.py\", line 575, in record_task_log_end\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     self.core_worker.record_task_log_end(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m AttributeError: 'Worker' object has no attribute 'core_worker'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1796, in ray._raylet.task_execution_handler\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1695, in ray._raylet.execute_task_with_cancellation_handler\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1380, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1595, in ray._raylet.execute_task\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 835, in ray._raylet.store_task_errors\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m AttributeError: 'Worker' object has no attribute 'core_worker'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1832, in ray._raylet.task_execution_handler\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/_private/utils.py\", line 182, in push_error_to_driver\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     worker.core_worker.push_error(job_id, error_type, message, time.time())\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m AttributeError: 'Worker' object has no attribute 'core_worker'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Exception ignored in: 'ray._raylet.task_execution_handler'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m Traceback (most recent call last):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"python/ray/_raylet.pyx\", line 1832, in ray._raylet.task_execution_handler\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/_private/utils.py\", line 182, in push_error_to_driver\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m     worker.core_worker.push_error(job_id, error_type, message, time.time())\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820220)\u001b[0m AttributeError: 'Worker' object has no attribute 'core_worker'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     eval_results = self._do_policy_eval(to_eval=to_eval)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/env_runner_v2.py\", line 1053, in _do_policy_eval\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     eval_results[policy_id] = policy.compute_actions_from_input_dict(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/policy/torch_policy.py\", line 325, in compute_actions_from_input_dict\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return self._compute_action_helper(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/threading.py\", line 24, in wrapper\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return func(self, *a, **k)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/policy/torch_policy.py\", line 987, in _compute_action_helper\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     dist_inputs, dist_class, state_out = self.action_distribution_fn(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/sac/sac_torch_policy.py\", line 165, in action_distribution_fn\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     action_dist_inputs, _ = model.get_action_model_outputs(model_out)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/sac/sac_torch_model.py\", line 317, in get_action_model_outputs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return self.action_model({\"obs\": model_out}, state_in, seq_lens)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     res = self.forward(restored, state or [], seq_lens)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/torch/fcnet.py\", line 146, in forward\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     self._features = self._hidden_layers(self._last_flat_in)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return forward_call(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/container.py\", line 217, in forward\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     input = module(input)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return forward_call(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/torch/misc.py\", line 259, in forward\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return self._model(x)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return forward_call(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/container.py\", line 217, in forward\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     input = module(input)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return forward_call(*args, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/linear.py\", line 114, in forward\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820217)\u001b[0m     return F.linear(input, self.weight, self.bias)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m     actions, logp = self.exploration.get_exploration_action(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/exploration/stochastic_sampling.py\", line 80, in get_exploration_action\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m     return self._get_torch_exploration_action(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/exploration/stochastic_sampling.py\", line 149, in _get_torch_exploration_action\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m     action = action_dist.sample()\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m     self.last_sample = self._squash(normal_sample)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/torch/torch_action_dist.py\", line 370, in _squash\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820210)\u001b[0m     squashed = ((torch.tanh(raw_values) + 1.0) / 2.0) * (\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m     return convert_to_numpy((actions, state_out, extra_fetches))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/numpy.py\", line 158, in convert_to_numpy\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m     return tree.map_structure(mapping, x)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/tree/__init__.py\", line 434, in map_structure\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m     return unflatten_as(structures[0],\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/tree/__init__.py\", line 367, in unflatten_as\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820218)\u001b[0m     if not is_nested(structure):\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820219)\u001b[0m     extra_fetches[SampleBatch.ACTION_PROB] = torch.exp(logp.float())\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     self._base_env.send_actions(actions_to_send)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/env/multi_agent_env.py\", line 645, in send_actions\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     obs, rewards, terminateds, truncateds, infos = env.step(agent_dict)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     self.env.step(action[self.env.agent_selection])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/environment/Environment.py\", line 536, in step\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     self.auction()\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/environment/Environment.py\", line 453, in auction\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     self.finished_job += self.parameters.auction_type.auction(self.request_job)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/environment/Auction.py\", line 25, in auction\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     winner_machine, prices, _ = self.allocation_mechanism.allocate(bids)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/environment/AllocationMechanism.py\", line 27, in allocate\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     winners = int(np.argsort(bids.bids)[0])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"<__array_function__ internals>\", line 180, in argsort\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/numpy/core/fromnumeric.py\", line 1120, in argsort\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     return _wrapfunc(a, 'argsort', axis=axis, kind=kind, order=order)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/numpy/core/fromnumeric.py\", line 54, in _wrapfunc\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     return _wrapit(obj, method, *args, **kwds)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/numpy/core/fromnumeric.py\", line 43, in _wrapit\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=820211)\u001b[0m     result = getattr(asarray(obj), method)(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import rllib_sac\n",
    "import ray\n",
    "\n",
    "ray.init()\n",
    "rllib_sac.train_sac(30,6)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "con = np.array([1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "con[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from environment import Environment\n",
    "def get_env_continuous(average_per_slot=50, machine_num=12):\n",
    "    para = Environment.VehicleJobSchedulingParameters(\n",
    "        average_per_slot=average_per_slot, machine_numbers=machine_num\n",
    "    )\n",
    "    para.action_space_continuous = True\n",
    "    env = Environment.VehicleJobSchedulingEnvACE(parameter=para)\n",
    "    return env\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rewards:\n",
      "{'Machine_0': 1692891.9449722767, 'Machine_1': 2753483.5262540476, 'Machine_2': 3115987.8549385746, 'Machine_3': 1578036.8233441114, 'Machine_4': 3589023.3625696828, 'Machine_5': 3015690.232190855}\n",
      "total reward:\n",
      "15745113.744269548\n"
     ]
    }
   ],
   "source": [
    "finished = False\n",
    "env = get_env_continuous(20,6)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total reward:\n",
      "16050359.294404507\n",
      "total reward:\n",
      "15439986.703701682\n",
      "total reward:\n",
      "15952020.87261987\n",
      "total reward:\n",
      "15802650.739851955\n",
      "total reward:\n",
      "15711688.175125126\n",
      "total reward:\n",
      "16194912.515216593\n",
      "total reward:\n",
      "16182266.693747701\n",
      "total reward:\n",
      "13536625.066564145\n",
      "total reward:\n",
      "15376459.682685792\n",
      "total reward:\n",
      "15812738.29275513\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    env.reset()\n",
    "    reward_sums = {a: 0.0 for a in env.possible_agents}\n",
    "    for agent in env.agent_iter():\n",
    "        observation, reward, termination, truncation, info = env.last()\n",
    "        reward_sums[agent] += reward\n",
    "        env.step(env.action_space(agent).sample())\n",
    "    print(\"total reward:\")\n",
    "    print(sum(reward_sums.values()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6771531717380093"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.finished_job/env.total_job\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max:  18486270.000000007\n",
      "min:  15925932.0\n",
      "mean:  17248282.740000002\n"
     ]
    }
   ],
   "source": [
    "env = get_env_continuous(20,6)\n",
    "episode_reward_max = 0\n",
    "episode_reward_min = np.inf\n",
    "episode_reward_mean = 0\n",
    "for i in range(50):\n",
    "    env.reset()\n",
    "    reward_sums = {a: 0.0 for a in env.possible_agents}\n",
    "    for agent in env.agent_iter():\n",
    "        observation, reward, termination, truncation, info = env.last()\n",
    "        reward_sums[agent] += reward\n",
    "        env.step(1)\n",
    "    episode_reward = sum(reward_sums.values())\n",
    "    # print(\"total reward:\" + str(episode_reward))\n",
    "    if episode_reward > episode_reward_max:\n",
    "        episode_reward_max = episode_reward\n",
    "    if episode_reward < episode_reward_min:\n",
    "        episode_reward_min = episode_reward\n",
    "    episode_reward_mean += episode_reward\n",
    "    \n",
    "print(\"max: \", episode_reward_max)\n",
    "print(\"min: \", episode_reward_min)\n",
    "print(\"mean: \", episode_reward_mean/50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5370744860128075"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.finished_job/env.total_job\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ray.tune import register_env\n",
    "\n",
    "import rllib_setup\n",
    "def create_env(machine,jobs):\n",
    "    env_name = \"VJS\"\n",
    "    alg_name = \"SAC\"\n",
    "    register_env(\n",
    "        env_name,\n",
    "        lambda config: rllib_setup.get_env_continuous(jobs, machine),\n",
    "    )\n",
    "    test_env = rllib_setup.get_env_continuous(jobs, machine)\n",
    "    return test_env\n",
    "\n",
    "test_env = create_env(12,60)\n",
    "\n",
    "test_env.reset()\n",
    "\n",
    "reward_sums = {a: 0.0 for a in test_env._agent_ids}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([ 72., 144.,  72., 144.,  72., 144.,  72., 144.,  72., 144.,  72.,\n",
      "       144.,  72., 144.,  72., 144.,  72., 144.,  72., 144.,  72., 144.,\n",
      "        72., 144.,  72., 144.,  72., 144.,  72., 144.,  72., 144.,  72.,\n",
      "       144.,  72., 144.,  72., 144.,  72., 144.,   0.,   0.,   1.,   0.,\n",
      "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
      "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   1.,   0.,   0.,   0.,\n",
      "         0.,   0.,   0.,   0.,   0.,   0.,   0.]), 0, False, False, {})\n",
      "SACTorchPolicy\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "mat1 and mat2 shapes cannot be multiplied (73x2 and 73x256)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m/home/yuan/ResMan/testApi.ipynb 单元格 40\u001b[0m line \u001b[0;36m1\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m policy \u001b[39m=\u001b[39m algo\u001b[39m.\u001b[39mget_policy(agent)\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m batch_obs \u001b[39m=\u001b[39m {\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mobs\u001b[39m\u001b[39m\"\u001b[39m: {\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mobservation\u001b[39m\u001b[39m\"\u001b[39m: observation,\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39maction_mask\u001b[39m\u001b[39m\"\u001b[39m: observation,\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m     }\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=14'>15</a>\u001b[0m }\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=15'>16</a>\u001b[0m batched_action, state_out, info \u001b[39m=\u001b[39m policy\u001b[39m.\u001b[39;49mcompute_actions_from_input_dict(\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=16'>17</a>\u001b[0m     batch_obs\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=17'>18</a>\u001b[0m )\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=18'>19</a>\u001b[0m single_action \u001b[39m=\u001b[39m batched_action[\u001b[39m0\u001b[39m]\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X54sdnNjb2RlLXJlbW90ZQ%3D%3D?line=19'>20</a>\u001b[0m action \u001b[39m=\u001b[39m single_action\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/policy/torch_policy.py:325\u001b[0m, in \u001b[0;36mTorchPolicy.compute_actions_from_input_dict\u001b[0;34m(self, input_dict, explore, timestep, **kwargs)\u001b[0m\n\u001b[1;32m    314\u001b[0m \u001b[39m# Calculate RNN sequence lengths.\u001b[39;00m\n\u001b[1;32m    315\u001b[0m seq_lens \u001b[39m=\u001b[39m (\n\u001b[1;32m    316\u001b[0m     torch\u001b[39m.\u001b[39mtensor(\n\u001b[1;32m    317\u001b[0m         [\u001b[39m1\u001b[39m] \u001b[39m*\u001b[39m \u001b[39mlen\u001b[39m(state_batches[\u001b[39m0\u001b[39m]),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    322\u001b[0m     \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    323\u001b[0m )\n\u001b[0;32m--> 325\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_compute_action_helper(\n\u001b[1;32m    326\u001b[0m     input_dict, state_batches, seq_lens, explore, timestep\n\u001b[1;32m    327\u001b[0m )\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/threading.py:24\u001b[0m, in \u001b[0;36mwith_lock.<locals>.wrapper\u001b[0;34m(self, *a, **k)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m     23\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_lock:\n\u001b[0;32m---> 24\u001b[0m         \u001b[39mreturn\u001b[39;00m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49ma, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mk)\n\u001b[1;32m     25\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mAttributeError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m     26\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mhas no attribute \u001b[39m\u001b[39m'\u001b[39m\u001b[39m_lock\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m e\u001b[39m.\u001b[39margs[\u001b[39m0\u001b[39m]:\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/policy/torch_policy.py:987\u001b[0m, in \u001b[0;36mTorchPolicy._compute_action_helper\u001b[0;34m(self, input_dict, state_batches, seq_lens, explore, timestep)\u001b[0m\n\u001b[1;32m    983\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maction_distribution_fn:\n\u001b[1;32m    984\u001b[0m     \u001b[39m# Try new action_distribution_fn signature, supporting\u001b[39;00m\n\u001b[1;32m    985\u001b[0m     \u001b[39m# state_batches and seq_lens.\u001b[39;00m\n\u001b[1;32m    986\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 987\u001b[0m         dist_inputs, dist_class, state_out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49maction_distribution_fn(\n\u001b[1;32m    988\u001b[0m             \u001b[39mself\u001b[39;49m,\n\u001b[1;32m    989\u001b[0m             \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel,\n\u001b[1;32m    990\u001b[0m             input_dict\u001b[39m=\u001b[39;49minput_dict,\n\u001b[1;32m    991\u001b[0m             state_batches\u001b[39m=\u001b[39;49mstate_batches,\n\u001b[1;32m    992\u001b[0m             seq_lens\u001b[39m=\u001b[39;49mseq_lens,\n\u001b[1;32m    993\u001b[0m             explore\u001b[39m=\u001b[39;49mexplore,\n\u001b[1;32m    994\u001b[0m             timestep\u001b[39m=\u001b[39;49mtimestep,\n\u001b[1;32m    995\u001b[0m             is_training\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    996\u001b[0m         )\n\u001b[1;32m    997\u001b[0m     \u001b[39m# Trying the old way (to stay backward compatible).\u001b[39;00m\n\u001b[1;32m    998\u001b[0m     \u001b[39m# TODO: Remove in future.\u001b[39;00m\n\u001b[1;32m    999\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mTypeError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/sac/sac_torch_policy.py:165\u001b[0m, in \u001b[0;36maction_distribution_fn\u001b[0;34m(policy, model, input_dict, state_batches, seq_lens, prev_action_batch, prev_reward_batch, explore, timestep, is_training)\u001b[0m\n\u001b[1;32m    162\u001b[0m model_out, _ \u001b[39m=\u001b[39m model(input_dict, [], \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m    163\u001b[0m \u001b[39m# Use the base output to get the policy outputs from the SAC model's\u001b[39;00m\n\u001b[1;32m    164\u001b[0m \u001b[39m# policy components.\u001b[39;00m\n\u001b[0;32m--> 165\u001b[0m action_dist_inputs, _ \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mget_action_model_outputs(model_out)\n\u001b[1;32m    166\u001b[0m \u001b[39m# Get a distribution class to be used with the just calculated dist-inputs.\u001b[39;00m\n\u001b[1;32m    167\u001b[0m action_dist_class \u001b[39m=\u001b[39m _get_dist_class(policy, policy\u001b[39m.\u001b[39mconfig, policy\u001b[39m.\u001b[39maction_space)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/sac/sac_torch_model.py:317\u001b[0m, in \u001b[0;36mSACTorchModel.get_action_model_outputs\u001b[0;34m(self, model_out, state_in, seq_lens)\u001b[0m\n\u001b[1;32m    315\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39maction_model\u001b[39m.\u001b[39mobs_space, Box):\n\u001b[1;32m    316\u001b[0m     model_out \u001b[39m=\u001b[39m concat_obs_if_necessary(model_out)\n\u001b[0;32m--> 317\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49maction_model({\u001b[39m\"\u001b[39;49m\u001b[39mobs\u001b[39;49m\u001b[39m\"\u001b[39;49m: model_out}, state_in, seq_lens)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/modelv2.py:259\u001b[0m, in \u001b[0;36mModelV2.__call__\u001b[0;34m(self, input_dict, state, seq_lens)\u001b[0m\n\u001b[1;32m    256\u001b[0m         restored[\u001b[39m\"\u001b[39m\u001b[39mobs_flat\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m input_dict[\u001b[39m\"\u001b[39m\u001b[39mobs\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m    258\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontext():\n\u001b[0;32m--> 259\u001b[0m     res \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(restored, state \u001b[39mor\u001b[39;49;00m [], seq_lens)\n\u001b[1;32m    261\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(input_dict, SampleBatch):\n\u001b[1;32m    262\u001b[0m     input_dict\u001b[39m.\u001b[39maccessed_keys \u001b[39m=\u001b[39m restored\u001b[39m.\u001b[39maccessed_keys \u001b[39m-\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39mobs_flat\u001b[39m\u001b[39m\"\u001b[39m}\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/torch/fcnet.py:146\u001b[0m, in \u001b[0;36mFullyConnectedNetwork.forward\u001b[0;34m(self, input_dict, state, seq_lens)\u001b[0m\n\u001b[1;32m    144\u001b[0m obs \u001b[39m=\u001b[39m input_dict[\u001b[39m\"\u001b[39m\u001b[39mobs_flat\u001b[39m\u001b[39m\"\u001b[39m]\u001b[39m.\u001b[39mfloat()\n\u001b[1;32m    145\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_last_flat_in \u001b[39m=\u001b[39m obs\u001b[39m.\u001b[39mreshape(obs\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m], \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m--> 146\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_hidden_layers(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_last_flat_in)\n\u001b[1;32m    147\u001b[0m logits \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_logits(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_logits \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features\n\u001b[1;32m    148\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfree_log_std:\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/container.py:217\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    215\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m    216\u001b[0m     \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 217\u001b[0m         \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m    218\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/models/torch/misc.py:259\u001b[0m, in \u001b[0;36mSlimFC.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    258\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x: TensorType) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m TensorType:\n\u001b[0;32m--> 259\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_model(x)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/container.py:217\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    215\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m    216\u001b[0m     \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 217\u001b[0m         \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m    218\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_pre_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[39mor\u001b[39;00m _global_backward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mlinear(\u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (73x2 and 73x256)"
     ]
    }
   ],
   "source": [
    "for agent in test_env.env.agent_iter():\n",
    "    observation, reward, termination, truncation, info = test_env.env.last()\n",
    "    print(test_env.env.last())\n",
    "    reward_sums[agent] += reward\n",
    "    if termination or truncation:\n",
    "        action = None\n",
    "    else:\n",
    "        print(algo.get_policy(agent))\n",
    "        policy = algo.get_policy(agent)\n",
    "        batch_obs = {\n",
    "            \"obs\": {\n",
    "                \"observation\": observation,\n",
    "                \"action_mask\": observation,\n",
    "            }\n",
    "        }\n",
    "        batched_action, state_out, info = policy.compute_actions_from_input_dict(\n",
    "            batch_obs\n",
    "        )\n",
    "        single_action = batched_action[0]\n",
    "        action = single_action\n",
    "\n",
    "    test_env.step(action)\n",
    "    i += 1\n",
    "    # env.render()\n",
    "\n",
    "print(\"rewards:\")\n",
    "print(reward_sums)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting performance benchmark\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11005.012010156304 turns per second\n",
      "917.0843341796921 cycles per second\n",
      "Finished performance benchmark\n",
      "Starting performance benchmark\n",
      "11391.133549607744 turns per second\n",
      "949.2611291339787 cycles per second\n",
      "Finished performance benchmark\n"
     ]
    }
   ],
   "source": [
    "from environment import environment_jax\n",
    "from environment import Environment\n",
    "from pettingzoo.test import performance_benchmark\n",
    "\n",
    "\n",
    "\n",
    "env = Environment.VehicleJobSchedulingEnvACE()\n",
    "env.reset()\n",
    "performance_benchmark(env)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting performance benchmark\n",
      "10990.468154029168 turns per second\n",
      "915.8723461690973 cycles per second\n",
      "Finished performance benchmark\n",
      "Starting performance benchmark\n",
      "7658.223707568317 turns per second\n",
      "638.1853089640264 cycles per second\n",
      "Finished performance benchmark\n",
      "         3769624 function calls (3615914 primitive calls) in 5.004 seconds\n",
      "\n",
      "   Ordered by: standard name\n",
      "\n",
      "   ncalls  tottime  percall  cumtime  percall filename:lineno(function)\n",
      "    36119    0.039    0.000    0.372    0.000 <__array_function__ internals>:177(all)\n",
      "     8178    0.008    0.000    0.133    0.000 <__array_function__ internals>:177(argsort)\n",
      "       33    0.000    0.000    0.000    0.000 <__array_function__ internals>:177(atleast_1d)\n",
      "       33    0.000    0.000    0.001    0.000 <__array_function__ internals>:177(broadcast_arrays)\n",
      "    38420    0.039    0.000    0.200    0.000 <__array_function__ internals>:177(concatenate)\n",
      "      108    0.000    0.000    0.001    0.000 <__array_function__ internals>:177(copyto)\n",
      "    34066    0.027    0.000    0.109    0.000 <__array_function__ internals>:177(dot)\n",
      "     8178    0.007    0.000    0.065    0.000 <__array_function__ internals>:177(sort)\n",
      "        1    0.000    0.000    5.004    5.004 <string>:1(<module>)\n",
      "       33    0.000    0.000    0.002    0.000 <string>:5(_parse_args_rvs)\n",
      "       64    0.000    0.000    0.000    0.000 __init__.py:97(_xla_gc_callback)\n",
      "       33    0.000    0.000    0.000    0.000 _discrete_distns.py:853(_argcheck)\n",
      "       33    0.000    0.000    0.002    0.000 _discrete_distns.py:856(_rvs)\n",
      "       33    0.001    0.000    0.008    0.000 _distn_infrastructure.py:1015(rvs)\n",
      "       33    0.000    0.000    0.008    0.000 _distn_infrastructure.py:3313(rvs)\n",
      "       33    0.001    0.000    0.002    0.000 _distn_infrastructure.py:867(_argcheck_rvs)\n",
      "       99    0.000    0.000    0.000    0.000 _distn_infrastructure.py:881(squeeze_left)\n",
      "       33    0.000    0.000    0.000    0.000 _distn_infrastructure.py:897(<listcomp>)\n",
      "       33    0.000    0.000    0.000    0.000 _distn_infrastructure.py:926(<listcomp>)\n",
      "       66    0.000    0.000    0.000    0.000 _methods.py:60(_all)\n",
      "       18    0.000    0.000    0.000    0.000 _ufunc_config.py:131(geterr)\n",
      "       18    0.000    0.000    0.000    0.000 _ufunc_config.py:32(seterr)\n",
      "        9    0.000    0.000    0.000    0.000 _ufunc_config.py:429(__enter__)\n",
      "        9    0.000    0.000    0.000    0.000 _ufunc_config.py:434(__exit__)\n",
      "       33    0.000    0.000    0.000    0.000 abc.py:117(__instancecheck__)\n",
      "     9004    0.006    0.000    0.012    0.000 agent_selector.py:12(__init__)\n",
      "     9004    0.006    0.000    0.006    0.000 agent_selector.py:15(reinit)\n",
      "    38313    0.035    0.000    0.042    0.000 agent_selector.py:24(next)\n",
      "    48064    0.017    0.000    0.017    0.000 agent_selector.py:29(is_last)\n",
      "    76822    0.011    0.000    0.011    0.000 box.py:141(is_np_flattenable)\n",
      "        9    0.000    0.000    0.000    0.000 contextlib.py:63(_recreate_cm)\n",
      "        9    0.000    0.000    0.001    0.000 contextlib.py:76(inner)\n",
      "    38411    0.048    0.000    0.203    0.000 dict.py:120(is_np_flattenable)\n",
      "   192055    0.083    0.000    0.104    0.000 dict.py:123(<genexpr>)\n",
      "    76822    0.010    0.000    0.010    0.000 discrete.py:42(is_np_flattenable)\n",
      "    38304    0.070    0.000    0.312    0.000 discrete.py:47(sample)\n",
      "     3192    0.002    0.000    0.002    0.000 env.py:135(num_agents)\n",
      "    38304    0.055    0.000    0.055    0.000 env.py:158(_clear_rewards)\n",
      "     9760    0.042    0.000    0.044    0.000 env.py:163(_accumulate_rewards)\n",
      "     3192    0.003    0.000    0.005    0.000 env.py:171(agent_iter)\n",
      "    38304    0.043    0.000    0.056    0.000 env.py:178(last)\n",
      "     3192    0.003    0.000    0.003    0.000 env.py:256(__init__)\n",
      "     3192    0.002    0.000    0.003    0.000 env.py:260(__iter__)\n",
      "     3192    0.001    0.000    0.001    0.000 env.py:265(__init__)\n",
      "    41496    0.030    0.000    0.030    0.000 env.py:269(__next__)\n",
      "    28548    0.007    0.000    0.007    0.000 environment_jax.py:1001(get_machine)\n",
      "     8182    0.020    0.000    0.716    0.000 environment_jax.py:1023(__init__)\n",
      "     8182    0.033    0.000    0.040    0.000 environment_jax.py:1024(<listcomp>)\n",
      "     8182    0.009    0.000    0.171    0.000 environment_jax.py:1030(get_bids)\n",
      "     8182    0.013    0.000    0.162    0.000 environment_jax.py:1031(<listcomp>)\n",
      "     8182    0.045    0.000    0.656    0.000 environment_jax.py:1033(request_bids)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1065(reset)\n",
      "       33    0.017    0.001    0.547    0.017 environment_jax.py:1070(generate_restrict)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1096(__iter__)\n",
      "       33    0.000    0.000    0.547    0.017 environment_jax.py:1099(__next__)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1104(__init__)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1109(__iter__)\n",
      "      814    0.002    0.000    0.550    0.001 environment_jax.py:1112(__next__)\n",
      "     8178    0.035    0.000    0.235    0.000 environment_jax.py:1200(allocate)\n",
      "     8182    0.052    0.000    1.411    0.000 environment_jax.py:1238(auction)\n",
      "     9954    0.018    0.000    0.160    0.000 environment_jax.py:126(__init__)\n",
      "        9    0.000    0.000    0.009    0.001 environment_jax.py:1390(reset)\n",
      "      813    0.000    0.000    0.000    0.000 environment_jax.py:1412(stop_condition_time_step)\n",
      "        9    0.000    0.000    0.017    0.002 environment_jax.py:1617(reset)\n",
      "     9954    0.033    0.000    0.142    0.000 environment_jax.py:162(calculate_budget)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1620(<listcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1623(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1624(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1625(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1626(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1627(<dictcomp>)\n",
      "        9    0.000    0.000    0.005    0.001 environment_jax.py:1628(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1634(<dictcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1643(<listcomp>)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:1644(<dictcomp>)\n",
      "    38304    0.013    0.000    0.013    0.000 environment_jax.py:1649(observe)\n",
      "     9004    0.006    0.000    0.555    0.000 environment_jax.py:1682(get_job_next_step)\n",
      "     9004    0.020    0.000    0.069    0.000 environment_jax.py:1689(_agent_selector)\n",
      "     8183    0.037    0.000    0.037    0.000 environment_jax.py:1692(<listcomp>)\n",
      "     8995    0.014    0.000    1.425    0.000 environment_jax.py:1699(auction)\n",
      "     8995    0.218    0.000    2.563    0.000 environment_jax.py:1703(next_job)\n",
      "      813    0.024    0.000    0.394    0.000 environment_jax.py:1721(<dictcomp>)\n",
      "      813    0.004    0.000    0.077    0.000 environment_jax.py:1734(round_end)\n",
      "        8    0.000    0.000    0.000    0.000 environment_jax.py:1742(<dictcomp>)\n",
      "        8    0.000    0.000    0.000    0.000 environment_jax.py:1743(<dictcomp>)\n",
      "    38304    0.044    0.000    0.044    0.000 environment_jax.py:1749(action)\n",
      "    38304    0.164    0.000    4.354    0.000 environment_jax.py:1759(step)\n",
      "     7505    0.025    0.000    0.027    0.000 environment_jax.py:182(start)\n",
      "     7505    0.003    0.000    0.003    0.000 environment_jax.py:188(finish)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:282(reset)\n",
      "       33    0.026    0.001    0.437    0.013 environment_jax.py:312(get_job_collections)\n",
      "        9    0.000    0.000    0.000    0.000 environment_jax.py:337(__iter__)\n",
      "       33    0.000    0.000    0.437    0.013 environment_jax.py:340(__next__)\n",
      "      108    0.000    0.000    0.000    0.000 environment_jax.py:350(__init__)\n",
      "      108    0.000    0.000    0.000    0.000 environment_jax.py:436(__init__)\n",
      "      108    0.007    0.000    0.008    0.000 environment_jax.py:608(reset)\n",
      "    24112    0.023    0.000    0.149    0.000 environment_jax.py:617(get_bid)\n",
      "    24112    0.048    0.000    0.126    0.000 environment_jax.py:623(drl_bid)\n",
      "     9954    0.011    0.000    0.040    0.000 environment_jax.py:63(priority_dist)\n",
      "     9756    0.002    0.000    0.002    0.000 environment_jax.py:632(clear_job)\n",
      "    38411    0.118    0.000    0.222    0.000 environment_jax.py:635(observe)\n",
      "    24112    0.009    0.000    0.009    0.000 environment_jax.py:681(request_auction)\n",
      "    28548    0.308    0.000    0.598    0.000 environment_jax.py:686(can_allocate)\n",
      "     7505    0.125    0.000    0.235    0.000 environment_jax.py:711(allocate_job)\n",
      "     9756    0.058    0.000    0.061    0.000 environment_jax.py:751(step)\n",
      "     9954    0.052    0.000    0.201    0.000 environment_jax.py:79(bi_model_dist)\n",
      "      813    0.001    0.000    0.066    0.000 environment_jax.py:981(step)\n",
      "      813    0.004    0.000    0.065    0.000 environment_jax.py:982(<listcomp>)\n",
      "      813    0.001    0.000    0.003    0.000 environment_jax.py:986(get_finish_job_total)\n",
      "      813    0.001    0.000    0.001    0.000 environment_jax.py:987(<listcomp>)\n",
      "      813    0.002    0.000    0.004    0.000 environment_jax.py:992(clear_job)\n",
      "        9    0.000    0.000    0.009    0.001 environment_jax.py:996(reset)\n",
      "     8178    0.002    0.000    0.002    0.000 fromnumeric.py:1008(_argsort_dispatcher)\n",
      "     8178    0.010    0.000    0.117    0.000 fromnumeric.py:1012(argsort)\n",
      "    36119    0.008    0.000    0.008    0.000 fromnumeric.py:2399(_all_dispatcher)\n",
      "    36119    0.047    0.000    0.285    0.000 fromnumeric.py:2404(all)\n",
      "     8178    0.032    0.000    0.086    0.000 fromnumeric.py:38(_wrapit)\n",
      "     8178    0.018    0.000    0.106    0.000 fromnumeric.py:51(_wrapfunc)\n",
      "    36119    0.072    0.000    0.238    0.000 fromnumeric.py:69(_wrapreduction)\n",
      "    36119    0.020    0.000    0.020    0.000 fromnumeric.py:70(<dictcomp>)\n",
      "     8178    0.002    0.000    0.002    0.000 fromnumeric.py:848(_sort_dispatcher)\n",
      "     8178    0.014    0.000    0.050    0.000 fromnumeric.py:852(sort)\n",
      "   192055    0.123    0.000    0.223    0.000 functools.py:817(dispatch)\n",
      "192055/38411    0.177    0.000    1.398    0.000 functools.py:883(wrapper)\n",
      "        1    0.000    0.000    0.000    0.000 iostream.py:203(schedule)\n",
      "        8    0.000    0.000    0.000    0.000 iostream.py:444(_is_master_process)\n",
      "        8    0.000    0.000    0.000    0.000 iostream.py:465(_schedule_flush)\n",
      "        8    0.000    0.000    0.000    0.000 iostream.py:535(write)\n",
      "        1    0.000    0.000    0.000    0.000 iostream.py:90(_event_pipe)\n",
      "      108    0.000    0.000    0.000    0.000 multiarray.py:1071(copyto)\n",
      "    38420    0.008    0.000    0.008    0.000 multiarray.py:148(concatenate)\n",
      "    34066    0.006    0.000    0.006    0.000 multiarray.py:736(dot)\n",
      "      108    0.000    0.000    0.001    0.000 numeric.py:149(ones)\n",
      "       33    0.000    0.000    0.000    0.000 numeric.py:1859(isscalar)\n",
      "        1    0.185    0.185    5.004    5.004 performance_benchmark.py:7(performance_benchmark)\n",
      "        9    0.000    0.000    0.000    0.000 random.py:791(getrandbits)\n",
      "       33    0.000    0.000    0.000    0.000 shape_base.py:19(_atleast_1d_dispatcher)\n",
      "       33    0.000    0.000    0.000    0.000 shape_base.py:23(atleast_1d)\n",
      "        1    0.000    0.000    0.000    0.000 socket.py:613(send)\n",
      "    38304    0.010    0.000    0.010    0.000 space.py:72(np_random)\n",
      "       33    0.001    0.000    0.001    0.000 stride_tricks.py:415(_broadcast_shape)\n",
      "       33    0.000    0.000    0.000    0.000 stride_tricks.py:475(_broadcast_arrays_dispatcher)\n",
      "       33    0.000    0.000    0.001    0.000 stride_tricks.py:479(broadcast_arrays)\n",
      "       33    0.000    0.000    0.000    0.000 stride_tricks.py:537(<listcomp>)\n",
      "      132    0.000    0.000    0.000    0.000 stride_tricks.py:541(<genexpr>)\n",
      "        1    0.000    0.000    0.000    0.000 threading.py:1066(_wait_for_tstate_lock)\n",
      "        1    0.000    0.000    0.000    0.000 threading.py:1133(is_alive)\n",
      "        1    0.000    0.000    0.000    0.000 threading.py:536(is_set)\n",
      "    76822    0.065    0.000    0.225    0.000 utils.py:138(_flatten_box_multibinary)\n",
      "    76822    0.068    0.000    0.164    0.000 utils.py:144(_flatten_discrete)\n",
      "    38411    0.108    0.000    1.275    0.000 utils.py:170(_flatten_dict)\n",
      "    38411    0.091    0.000    0.757    0.000 utils.py:173(<listcomp>)\n",
      "   192055    0.077    0.000    0.077    0.000 weakref.py:415(__getitem__)\n",
      "       33    0.000    0.000    0.000    0.000 {built-in method _abc._abc_instancecheck}\n",
      "   192055    0.024    0.000    0.024    0.000 {built-in method _abc.get_cache_token}\n",
      "   115077    0.059    0.000    0.163    0.000 {built-in method builtins.all}\n",
      "        1    0.000    0.000    5.004    5.004 {built-in method builtins.exec}\n",
      "    16422    0.005    0.000    0.005    0.000 {built-in method builtins.getattr}\n",
      "    38411    0.012    0.000    0.012    0.000 {built-in method builtins.isinstance}\n",
      "       18    0.000    0.000    0.000    0.000 {built-in method builtins.iter}\n",
      "    58795    0.011    0.000    0.011    0.000 {built-in method builtins.len}\n",
      "     9954    0.008    0.000    0.008    0.000 {built-in method builtins.max}\n",
      "9061/8995    0.005    0.000    0.559    0.000 {built-in method builtins.next}\n",
      "        4    0.000    0.000    0.000    0.000 {built-in method builtins.print}\n",
      "      813    0.001    0.000    0.001    0.000 {built-in method builtins.sum}\n",
      "        9    0.000    0.000    0.000    0.000 {built-in method from_bytes}\n",
      "       64    0.000    0.000    0.000    0.000 {built-in method jaxlib.xla_extension.collect_garbage}\n",
      "     9954    0.011    0.000    0.011    0.000 {built-in method numpy.arange}\n",
      "    96829    0.121    0.000    0.121    0.000 {built-in method numpy.array}\n",
      "     8211    0.010    0.000    0.010    0.000 {built-in method numpy.asanyarray}\n",
      "    85000    0.078    0.000    0.078    0.000 {built-in method numpy.asarray}\n",
      "   125135    0.283    0.000    0.735    0.000 {built-in method numpy.core._multiarray_umath.implement_array_function}\n",
      "      108    0.000    0.000    0.000    0.000 {built-in method numpy.empty}\n",
      "       36    0.000    0.000    0.000    0.000 {built-in method numpy.geterrobj}\n",
      "       18    0.000    0.000    0.000    0.000 {built-in method numpy.seterrobj}\n",
      "    86776    0.102    0.000    0.102    0.000 {built-in method numpy.zeros}\n",
      "        8    0.000    0.000    0.000    0.000 {built-in method posix.getpid}\n",
      "        9    0.000    0.000    0.000    0.000 {built-in method posix.urandom}\n",
      "     3194    0.001    0.000    0.001    0.000 {built-in method time.time}\n",
      "        9    0.001    0.000    0.001    0.000 {function SeedSequence.generate_state at 0x7feac81db550}\n",
      "        8    0.000    0.000    0.000    0.000 {method '__exit__' of '_thread.RLock' objects}\n",
      "        1    0.000    0.000    0.000    0.000 {method 'acquire' of '_thread.lock' objects}\n",
      "       66    0.000    0.000    0.001    0.000 {method 'all' of 'numpy.generic' objects}\n",
      "        1    0.000    0.000    0.000    0.000 {method 'append' of 'collections.deque' objects}\n",
      "    49671    0.009    0.000    0.009    0.000 {method 'append' of 'list' objects}\n",
      "     8178    0.030    0.000    0.030    0.000 {method 'argsort' of 'numpy.ndarray' objects}\n",
      "       33    0.000    0.000    0.000    0.000 {method 'astype' of 'numpy.ndarray' objects}\n",
      "     8178    0.013    0.000    0.013    0.000 {method 'copy' of 'numpy.ndarray' objects}\n",
      "        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
      "    76822    0.102    0.000    0.102    0.000 {method 'flatten' of 'numpy.ndarray' objects}\n",
      "       33    0.000    0.000    0.000    0.000 {method 'get' of 'dict' objects}\n",
      "    38304    0.232    0.000    0.232    0.000 {method 'integers' of 'numpy.random._generator.Generator' objects}\n",
      "    38411    0.006    0.000    0.006    0.000 {method 'items' of 'collections.OrderedDict' objects}\n",
      "    45879    0.009    0.000    0.009    0.000 {method 'items' of 'dict' objects}\n",
      "        9    0.000    0.000    0.000    0.000 {method 'keys' of 'dict' objects}\n",
      "     9954    0.053    0.000    0.053    0.000 {method 'normal' of 'numpy.random.mtrand.RandomState' objects}\n",
      "       33    0.002    0.000    0.002    0.000 {method 'poisson' of 'numpy.random.mtrand.RandomState' objects}\n",
      "       66    0.000    0.000    0.000    0.000 {method 'pop' of 'dict' objects}\n",
      "     9954    0.009    0.000    0.009    0.000 {method 'rand' of 'numpy.random.mtrand.RandomState' objects}\n",
      "    69678    0.222    0.000    0.222    0.000 {method 'randint' of 'numpy.random.mtrand.RandomState' objects}\n",
      "    36119    0.138    0.000    0.138    0.000 {method 'reduce' of 'numpy.ufunc' objects}\n",
      "     7077    0.002    0.000    0.002    0.000 {method 'remove' of 'list' objects}\n",
      "       33    0.000    0.000    0.000    0.000 {method 'reshape' of 'numpy.ndarray' objects}\n",
      "        9    0.001    0.000    0.002    0.000 {method 'seed' of 'numpy.random.mtrand.RandomState' objects}\n",
      "     9954    0.022    0.000    0.022    0.000 {method 'shuffle' of 'numpy.random.mtrand.RandomState' objects}\n",
      "     8178    0.013    0.000    0.013    0.000 {method 'sort' of 'numpy.ndarray' objects}\n",
      "    38411    0.007    0.000    0.007    0.000 {method 'values' of 'collections.OrderedDict' objects}\n",
      "    76600    0.011    0.000    0.011    0.000 {method 'values' of 'dict' objects}\n",
      "        8    0.000    0.000    0.000    0.000 {method 'write' of '_io.StringIO' objects}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import cProfile\n",
    "env = environment_jax.VehicleJobSchedulingEnvACE()\n",
    "env.reset()\n",
    "cProfile.run(\"performance_benchmark(env)\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Array(100, dtype=int32)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import jax.numpy as np\n",
    "np.max(np.array([24, 100]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "\n",
    "import ray\n",
    "from ray import tune\n",
    "from ray.rllib.algorithms.ppo import PPOConfig\n",
    "from ray.rllib.policy.policy import PolicySpec\n",
    "from ray.tune import register_env\n",
    "\n",
    "import rllib_setup\n",
    "\n",
    "\n",
    "def train(jobs, machine, rollout_workers=10):\n",
    "    env_name = \"VJS\"\n",
    "    alg_name = \"PPO\"\n",
    "    register_env(\n",
    "        env_name,\n",
    "        lambda config: rllib_setup.get_env_continuous(jobs, machine),\n",
    "    )\n",
    "    test_env = rllib_setup.get_env_continuous(jobs, machine)\n",
    "\n",
    "    def policies(agent_ids):\n",
    "        obs_space = test_env.observation_space\n",
    "        act_space = test_env.action_space\n",
    "        return {\n",
    "            str(i): (\n",
    "                None,\n",
    "                obs_space,\n",
    "                act_space,\n",
    "                {}\n",
    "                # config=config.overrides(agent_id=int(i[8:])),\n",
    "            )\n",
    "            for i in agent_ids\n",
    "        }\n",
    "\n",
    "    config = (\n",
    "        PPOConfig()\n",
    "        .rollouts(num_rollout_workers=rollout_workers)\n",
    "        .training(vf_clip_param=1080)\n",
    "        .resources(num_gpus=2)\n",
    "        .multi_agent(\n",
    "            policies=policies(test_env._agent_ids),\n",
    "            policy_mapping_fn=lambda agent_id, episode, **kwargs: str(agent_id),\n",
    "        )\n",
    "        .environment(env=env_name, disable_env_checking=True)\n",
    "        .evaluation(evaluation_interval=10)\n",
    "    )\n",
    "    config.model= {\n",
    "                \"fcnet_hiddens\": [384, 256, 128],\n",
    "                \"fcnet_activation\": \"relu\",\n",
    "                \"use_lstm\": True,\n",
    "                \"lstm_cell_size\": 256,\n",
    "            }\n",
    "    # print(config.to_dict())\n",
    "    # Build a Algorithm object from the config and run one training iteration.\n",
    "    # algo = config.build(env=env_name)\n",
    "    config.reuse_actors = True\n",
    "    tune.run(\n",
    "        alg_name,\n",
    "        name=\"PPO\" + str(machine) + \"_\" + str(jobs) + \"_lstm\",\n",
    "        stop={\"episodes_total\": 10000},\n",
    "        checkpoint_freq=10,\n",
    "        config=config.to_dict(),\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fcnet_hiddens': [384, 256, 128],\n",
       " 'fcnet_activation': 'relu',\n",
       " 'use_lstm': True,\n",
       " 'lstm_cell_size': 256}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-10 09:26:14,676\tWARNING algorithm_config.py:643 -- Cannot create PPOConfig from given `config_dict`! Property reuse_actors not supported.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(PPO pid=2395436)\u001b[0m 2023-10-10 09:26:20,616\tWARNING algorithm_config.py:643 -- Cannot create PPOConfig from given `config_dict`! Property reuse_actors not supported.\n",
      "\u001b[2m\u001b[36m(PPO pid=2395436)\u001b[0m 2023-10-10 09:26:20,633\tINFO algorithm.py:536 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
      "\u001b[2m\u001b[36m(PPO pid=2395436)\u001b[0m 2023-10-10 09:26:33,744\tINFO trainable.py:173 -- Trainable.setup took 13.113 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name         </th><th style=\"text-align: right;\">  agent_timesteps_total</th><th>connector_metrics  </th><th>counters                                                                                                                            </th><th>custom_metrics  </th><th>date               </th><th>done  </th><th style=\"text-align: right;\">  episode_len_mean</th><th>episode_media  </th><th style=\"text-align: right;\">  episode_reward_max</th><th style=\"text-align: right;\">  episode_reward_mean</th><th style=\"text-align: right;\">  episode_reward_min</th><th style=\"text-align: right;\">  episodes_this_iter</th><th style=\"text-align: right;\">  episodes_total</th><th>evaluation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               </th><th>hostname        </th><th>info                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip    </th><th style=\"text-align: right;\">  num_agent_steps_sampled</th><th style=\"text-align: right;\">  num_agent_steps_trained</th><th style=\"text-align: right;\">  num_env_steps_sampled</th><th style=\"text-align: right;\">  num_env_steps_sampled_this_iter</th><th style=\"text-align: right;\">  num_env_steps_sampled_throughput_per_sec</th><th style=\"text-align: right;\">  num_env_steps_trained</th><th style=\"text-align: right;\">  num_env_steps_trained_this_iter</th><th style=\"text-align: right;\">  num_env_steps_trained_throughput_per_sec</th><th style=\"text-align: right;\">  num_faulty_episodes</th><th style=\"text-align: right;\">  num_healthy_workers</th><th style=\"text-align: right;\">  num_in_flight_async_reqs</th><th style=\"text-align: right;\">  num_remote_worker_restarts</th><th style=\"text-align: right;\">  num_steps_trained_this_iter</th><th>perf                                                                                                                                                                                                                                                      </th><th style=\"text-align: right;\">    pid</th><th>policy_reward_max  </th><th>policy_reward_mean  </th><th>policy_reward_min  </th><th>sampler_perf  </th><th>sampler_results                                                                                                                                                                                                                                                                                                                                                                                </th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th>timers                                                                                                                                                                                                               </th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_total</th><th style=\"text-align: right;\">  training_iteration</th><th>trial_id   </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>PPO_VJS_00d92_00000</td><td style=\"text-align: right;\">                  39880</td><td>{}                 </td><td>{&#x27;num_env_steps_sampled&#x27;: 40000, &#x27;num_env_steps_trained&#x27;: 40000, &#x27;num_agent_steps_sampled&#x27;: 39880, &#x27;num_agent_steps_trained&#x27;: 39880}</td><td>{}              </td><td>2023-10-10_09-47-08</td><td>False </td><td style=\"text-align: right;\">               nan</td><td>{}             </td><td style=\"text-align: right;\">                 nan</td><td style=\"text-align: right;\">                  nan</td><td style=\"text-align: right;\">                 nan</td><td style=\"text-align: right;\">                   0</td><td style=\"text-align: right;\">               0</td><td>{&#x27;sampler_results&#x27;: {&#x27;episode_reward_max&#x27;: 17127754.023542404, &#x27;episode_reward_min&#x27;: 14082027.171352029, &#x27;episode_reward_mean&#x27;: 15564772.212167716, &#x27;episode_len_mean&#x27;: 7570.0, &#x27;episode_media&#x27;: {}, &#x27;episodes_this_iter&#x27;: 10, &#x27;policy_reward_min&#x27;: {&#x27;Machine_3&#x27;: 1201122.6355660558, &#x27;Machine_2&#x27;: 2240558.163669421, &#x27;Machine_4&#x27;: 4592997.810513791, &#x27;Machine_1&#x27;: 3526818.5287414733, &#x27;Machine_5&#x27;: 407695.7844722981, &#x27;Machine_0&#x27;: 1121932.8467915654}, &#x27;policy_reward_max&#x27;: {&#x27;Machine_3&#x27;: 1770023.79865551, &#x27;Machine_2&#x27;: 2934755.4741453025, &#x27;Machine_4&#x27;: 5728531.543715833, &#x27;Machine_1&#x27;: 5007355.348616596, &#x27;Machine_5&#x27;: 1043279.2218626284, &#x27;Machine_0&#x27;: 1921423.0442111492}, &#x27;policy_reward_mean&#x27;: {&#x27;Machine_3&#x27;: 1507767.7724013268, &#x27;Machine_2&#x27;: 2614883.9808027497, &#x27;Machine_4&#x27;: 5187074.866675732, &#x27;Machine_1&#x27;: 4158338.1460392848, &#x27;Machine_5&#x27;: 609283.3278170584, &#x27;Machine_0&#x27;: 1487424.11843158}, &#x27;custom_metrics&#x27;: {}, &#x27;hist_stats&#x27;: {&#x27;episode_reward&#x27;: [14728026.913376212, 15474842.345697105, 15679472.080604374, 15483649.373023152, 15871152.510795414, 14082027.171352029, 16192273.711469652, 15769218.216767907, 15239305.775048912, 17127754.023542404], &#x27;episode_lengths&#x27;: [7223, 7531, 7975, 7759, 7457, 7273, 7817, 7664, 7496, 7505], &#x27;policy_Machine_3_reward&#x27;: [1417239.6961083412, 1544024.706980288, 1770023.79865551, 1308140.7149455547, 1657978.6802791357, 1201122.6355660558, 1533451.2038691044, 1531127.0858809948, 1642113.977706492, 1472455.2240217924], &#x27;policy_Machine_2_reward&#x27;: [2380880.31442119, 2240558.163669421, 2670592.6217448222, 2552399.901468944, 2693219.312130402, 2318004.6872558724, 2934755.4741453025, 2670143.52125312, 2772676.4761321656, 2915609.3358062585], &#x27;policy_Machine_4_reward&#x27;: [5208943.7219518395, 5290380.472485718, 5113872.876039084, 5459750.319732364, 5728531.543715833, 4592997.810513791, 4884875.815126177, 5152847.031736787, 5211913.949671921, 5226635.1257838], &#x27;policy_Machine_1_reward&#x27;: [3589658.6771131107, 4342813.497587556, 4225788.475040789, 4319368.358900185, 4182454.6543533755, 3834323.871371024, 4142203.407575304, 4412596.641093429, 3526818.5287414733, 5007355.348616596], &#x27;policy_Machine_5_reward&#x27;: [550011.0580716843, 517585.69159169105, 483657.2857046835, 407695.7844722981, 487035.47352511843, 709798.8885597113, 775564.7665426256, 438304.47164733335, 679900.636192811, 1043279.2218626284], &#x27;policy_Machine_0_reward&#x27;: [1581293.445710063, 1539479.8133824468, 1415537.0234194994, 1436294.293503821, 1121932.8467915654, 1425779.2780855894, 1921423.0442111492, 1564199.4651562572, 1405882.2066040635, 1462419.767451346]}, &#x27;sampler_perf&#x27;: {&#x27;mean_raw_obs_processing_ms&#x27;: 0.4218998560777777, &#x27;mean_inference_ms&#x27;: 8.53638778367343, &#x27;mean_action_processing_ms&#x27;: 0.24125076543686266, &#x27;mean_env_wait_ms&#x27;: 0.12995032038635562, &#x27;mean_env_render_ms&#x27;: 0.0}, &#x27;num_faulty_episodes&#x27;: 0, &#x27;connector_metrics&#x27;: {&#x27;ObsPreprocessorConnector_ms&#x27;: 0.0058333079020182295, &#x27;StateBufferConnector_ms&#x27;: 0.007167657216389974, &#x27;ViewRequirementAgentConnector_ms&#x27;: 0.19806226094563803}}, &#x27;episode_reward_max&#x27;: 17127754.023542404, &#x27;episode_reward_min&#x27;: 14082027.171352029, &#x27;episode_reward_mean&#x27;: 15564772.212167716, &#x27;episode_len_mean&#x27;: 7570.0, &#x27;episode_media&#x27;: {}, &#x27;episodes_this_iter&#x27;: 10, &#x27;policy_reward_min&#x27;: {&#x27;Machine_3&#x27;: 1201122.6355660558, &#x27;Machine_2&#x27;: 2240558.163669421, &#x27;Machine_4&#x27;: 4592997.810513791, &#x27;Machine_1&#x27;: 3526818.5287414733, &#x27;Machine_5&#x27;: 407695.7844722981, &#x27;Machine_0&#x27;: 1121932.8467915654}, &#x27;policy_reward_max&#x27;: {&#x27;Machine_3&#x27;: 1770023.79865551, &#x27;Machine_2&#x27;: 2934755.4741453025, &#x27;Machine_4&#x27;: 5728531.543715833, &#x27;Machine_1&#x27;: 5007355.348616596, &#x27;Machine_5&#x27;: 1043279.2218626284, &#x27;Machine_0&#x27;: 1921423.0442111492}, &#x27;policy_reward_mean&#x27;: {&#x27;Machine_3&#x27;: 1507767.7724013268, &#x27;Machine_2&#x27;: 2614883.9808027497, &#x27;Machine_4&#x27;: 5187074.866675732, &#x27;Machine_1&#x27;: 4158338.1460392848, &#x27;Machine_5&#x27;: 609283.3278170584, &#x27;Machine_0&#x27;: 1487424.11843158}, &#x27;custom_metrics&#x27;: {}, &#x27;hist_stats&#x27;: {&#x27;episode_reward&#x27;: [14728026.913376212, 15474842.345697105, 15679472.080604374, 15483649.373023152, 15871152.510795414, 14082027.171352029, 16192273.711469652, 15769218.216767907, 15239305.775048912, 17127754.023542404], &#x27;episode_lengths&#x27;: [7223, 7531, 7975, 7759, 7457, 7273, 7817, 7664, 7496, 7505], &#x27;policy_Machine_3_reward&#x27;: [1417239.6961083412, 1544024.706980288, 1770023.79865551, 1308140.7149455547, 1657978.6802791357, 1201122.6355660558, 1533451.2038691044, 1531127.0858809948, 1642113.977706492, 1472455.2240217924], &#x27;policy_Machine_2_reward&#x27;: [2380880.31442119, 2240558.163669421, 2670592.6217448222, 2552399.901468944, 2693219.312130402, 2318004.6872558724, 2934755.4741453025, 2670143.52125312, 2772676.4761321656, 2915609.3358062585], &#x27;policy_Machine_4_reward&#x27;: [5208943.7219518395, 5290380.472485718, 5113872.876039084, 5459750.319732364, 5728531.543715833, 4592997.810513791, 4884875.815126177, 5152847.031736787, 5211913.949671921, 5226635.1257838], &#x27;policy_Machine_1_reward&#x27;: [3589658.6771131107, 4342813.497587556, 4225788.475040789, 4319368.358900185, 4182454.6543533755, 3834323.871371024, 4142203.407575304, 4412596.641093429, 3526818.5287414733, 5007355.348616596], &#x27;policy_Machine_5_reward&#x27;: [550011.0580716843, 517585.69159169105, 483657.2857046835, 407695.7844722981, 487035.47352511843, 709798.8885597113, 775564.7665426256, 438304.47164733335, 679900.636192811, 1043279.2218626284], &#x27;policy_Machine_0_reward&#x27;: [1581293.445710063, 1539479.8133824468, 1415537.0234194994, 1436294.293503821, 1121932.8467915654, 1425779.2780855894, 1921423.0442111492, 1564199.4651562572, 1405882.2066040635, 1462419.767451346]}, &#x27;sampler_perf&#x27;: {&#x27;mean_raw_obs_processing_ms&#x27;: 0.4218998560777777, &#x27;mean_inference_ms&#x27;: 8.53638778367343, &#x27;mean_action_processing_ms&#x27;: 0.24125076543686266, &#x27;mean_env_wait_ms&#x27;: 0.12995032038635562, &#x27;mean_env_render_ms&#x27;: 0.0}, &#x27;num_faulty_episodes&#x27;: 0, &#x27;connector_metrics&#x27;: {&#x27;ObsPreprocessorConnector_ms&#x27;: 0.0058333079020182295, &#x27;StateBufferConnector_ms&#x27;: 0.007167657216389974, &#x27;ViewRequirementAgentConnector_ms&#x27;: 0.19806226094563803}, &#x27;num_agent_steps_sampled_this_iter&#x27;: 75700, &#x27;num_env_steps_sampled_this_iter&#x27;: 75700, &#x27;timesteps_this_iter&#x27;: 75700, &#x27;num_healthy_workers&#x27;: 0, &#x27;num_in_flight_async_reqs&#x27;: 0, &#x27;num_remote_worker_restarts&#x27;: 0}</td><td>lwh-Super-Server</td><td>{&#x27;learner&#x27;: {&#x27;Machine_0&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 2.2781249999999997, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 845.3849156697592, &#x27;policy_loss&#x27;: -0.12222361331805587, &#x27;vf_loss&#x27;: 845.4807154337565, &#x27;vf_explained_var&#x27;: -1.4445434014002482e-05, &#x27;kl&#x27;: 0.011595767002048282, &#x27;entropy&#x27;: 1.2730616261561711, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1410.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 59.5}, &#x27;Machine_1&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 0.04999999999999999, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 845.9146968587239, &#x27;policy_loss&#x27;: 0.036568332239985464, &#x27;vf_loss&#x27;: 845.8778076171875, &#x27;vf_explained_var&#x27;: 5.408128102620443e-07, &#x27;kl&#x27;: 0.006388123415720959, &#x27;entropy&#x27;: 1.5013228003184, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1305.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 74.5}, &#x27;Machine_2&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 0.6749999999999998, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 884.5687027994792, &#x27;policy_loss&#x27;: 0.007973820368448892, &#x27;vf_loss&#x27;: 884.5513496907552, &#x27;vf_explained_var&#x27;: 9.117523829142253e-06, &#x27;kl&#x27;: 0.013888688227743841, &#x27;entropy&#x27;: 2.4835897493362427, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1335.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 74.5}, &#x27;Machine_3&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 2.2781249999999993, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 885.0926444498698, &#x27;policy_loss&#x27;: -0.060079560180505115, &#x27;vf_loss&#x27;: 885.1309391276042, &#x27;vf_explained_var&#x27;: 1.7361243565877277e-05, &#x27;kl&#x27;: 0.009561835904605687, &#x27;entropy&#x27;: 2.3671712478001914, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1305.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 74.5}, &#x27;Machine_4&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 0.037500000000000006, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 886.4752482096354, &#x27;policy_loss&#x27;: 0.011554135108987491, &#x27;vf_loss&#x27;: 886.4633056640625, &#x27;vf_explained_var&#x27;: 2.396106719970703e-07, &#x27;kl&#x27;: 0.011143198707529034, &#x27;entropy&#x27;: 1.8122947716712952, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1395.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 74.5}, &#x27;Machine_5&#x27;: {&#x27;custom_metrics&#x27;: {}, &#x27;learner_stats&#x27;: {&#x27;cur_kl_coeff&#x27;: 0.11250000000000003, &#x27;cur_lr&#x27;: 5.0000000000000016e-05, &#x27;total_loss&#x27;: 881.9105494181315, &#x27;policy_loss&#x27;: -0.0997712252040704, &#x27;vf_loss&#x27;: 882.0088144938151, &#x27;vf_explained_var&#x27;: -0.0005312701066335042, &#x27;kl&#x27;: 0.013343523483005508, &#x27;entropy&#x27;: 0.9184814234574635, &#x27;entropy_coeff&#x27;: 0.0}, &#x27;model&#x27;: {}, &#x27;num_grad_updates_lifetime&#x27;: 1290.5, &#x27;diff_num_grad_updates_vs_sampler_policy&#x27;: 59.5}}, &#x27;num_env_steps_sampled&#x27;: 40000, &#x27;num_env_steps_trained&#x27;: 40000, &#x27;num_agent_steps_sampled&#x27;: 39880, &#x27;num_agent_steps_trained&#x27;: 39880}</td><td style=\"text-align: right;\">                        10</td><td>192.168.3.6</td><td style=\"text-align: right;\">                    39880</td><td style=\"text-align: right;\">                    39880</td><td style=\"text-align: right;\">                  40000</td><td style=\"text-align: right;\">                             4000</td><td style=\"text-align: right;\">                                   75.3406</td><td style=\"text-align: right;\">                  40000</td><td style=\"text-align: right;\">                             4000</td><td style=\"text-align: right;\">                                   75.3406</td><td style=\"text-align: right;\">                    0</td><td style=\"text-align: right;\">                   24</td><td style=\"text-align: right;\">                         0</td><td style=\"text-align: right;\">                           0</td><td style=\"text-align: right;\">                         4000</td><td>{&#x27;cpu_util_percent&#x27;: 8.540171990171991, &#x27;ram_util_percent&#x27;: 17.004791154791153, &#x27;gpu_util_percent0&#x27;: 0.7796928746928746, &#x27;vram_util_percent0&#x27;: 0.8375651041666666, &#x27;gpu_util_percent1&#x27;: 0.0014373464373464374, &#x27;vram_util_percent1&#x27;: 0.020833333333333332}</td><td style=\"text-align: right;\">2395436</td><td>{}                 </td><td>{}                  </td><td>{}                 </td><td>{}            </td><td>{&#x27;episode_reward_max&#x27;: nan, &#x27;episode_reward_min&#x27;: nan, &#x27;episode_reward_mean&#x27;: nan, &#x27;episode_len_mean&#x27;: nan, &#x27;episode_media&#x27;: {}, &#x27;episodes_this_iter&#x27;: 0, &#x27;policy_reward_min&#x27;: {}, &#x27;policy_reward_max&#x27;: {}, &#x27;policy_reward_mean&#x27;: {}, &#x27;custom_metrics&#x27;: {}, &#x27;hist_stats&#x27;: {&#x27;episode_reward&#x27;: [], &#x27;episode_lengths&#x27;: []}, &#x27;sampler_perf&#x27;: {}, &#x27;num_faulty_episodes&#x27;: 0, &#x27;connector_metrics&#x27;: {}}</td><td style=\"text-align: right;\">             1233.48</td><td style=\"text-align: right;\">           760.515</td><td style=\"text-align: right;\">       1233.48</td><td>{&#x27;training_iteration_time_ms&#x27;: 52594.712, &#x27;sample_time_ms&#x27;: 921.594, &#x27;load_time_ms&#x27;: 129.767, &#x27;load_throughput&#x27;: 30824.384, &#x27;learn_time_ms&#x27;: 51375.333, &#x27;learn_throughput&#x27;: 77.858, &#x27;synch_weights_time_ms&#x27;: 165.146}</td><td style=\"text-align: right;\"> 1696902428</td><td style=\"text-align: right;\">            40000</td><td style=\"text-align: right;\">                  10</td><td>00d92_00000</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(PPO pid=2395436)\u001b[0m 2023-10-10 09:36:18,844\tWARNING env_runner_v2.py:154 -- More than 7223 observations in 7223 env steps for episode 199955370593790409 are buffered in the sampler. If this is more than you expected, check that that you set a horizon on your environment correctly and that it terminates at some point. Note: In multi-agent environments, `rollout_fragment_length` sets the batch size based on (across-agents) environment steps, not the steps of individual agents, which can result in unexpectedly large batches.Also, you may be waiting for your Env to terminate (batch_mode=`complete_episodes`). Make sure it does at some point.\n"
     ]
    }
   ],
   "source": [
    "train(20,6,24)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/algorithm.py:442: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "`UnifiedLogger` will be removed in Ray 2.7.\n",
      "  return UnifiedLogger(config, logdir, loggers=None)\n",
      "/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `JsonLogger interface is deprecated in favor of the `ray.tune.json.JsonLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `CSVLogger interface is deprecated in favor of the `ray.tune.csv.CSVLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n",
      "The `TBXLogger interface is deprecated in favor of the `ray.tune.tensorboardx.TBXLoggerCallback` interface and will be removed in Ray 2.7.\n",
      "  self._loggers.append(cls(self.config, self.logdir, self.trial))\n",
      "2023-10-10 10:05:28,632\tINFO worker.py:1627 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8265 \u001b[39m\u001b[22m\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m 2023-10-10 10:05:36,553\tERROR worker.py:861 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554711, ip=192.168.3.6, actor_id=9d0e579dc9a8323e7320d29f01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7faead2ab760>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m     self._update_policy_map(policy_dict=self.policy_dict)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m     self._build_policy_map(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m     new_policy = create_policy_for_framework(\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m     return policy_class(observation_space, action_space, merged_config)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m   File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m     max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=2554711)\u001b[0m KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,869\tERROR actor_manager.py:507 -- Ray error, taking actor 1 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554705, ip=192.168.3.6, actor_id=67126442aae83852ab86beec01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f880a30f700>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,872\tERROR actor_manager.py:507 -- Ray error, taking actor 2 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554706, ip=192.168.3.6, actor_id=1b6abe028f65b2d856c9d7a301000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f141b68b730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,875\tERROR actor_manager.py:507 -- Ray error, taking actor 3 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554707, ip=192.168.3.6, actor_id=f3e17586b0b5c16cc7961a0d01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f5a97681730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,877\tERROR actor_manager.py:507 -- Ray error, taking actor 4 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554708, ip=192.168.3.6, actor_id=80ee5bfd570038e57fea0c4001000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f788b28b6d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,880\tERROR actor_manager.py:507 -- Ray error, taking actor 5 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554709, ip=192.168.3.6, actor_id=8f8a30b206791ce6b6d0328501000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f1cb546c6d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,882\tERROR actor_manager.py:507 -- Ray error, taking actor 6 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554710, ip=192.168.3.6, actor_id=2674e13011d1637553bd522201000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fa9d140b700>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,885\tERROR actor_manager.py:507 -- Ray error, taking actor 7 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554711, ip=192.168.3.6, actor_id=9d0e579dc9a8323e7320d29f01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7faead2ab760>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,888\tERROR actor_manager.py:507 -- Ray error, taking actor 8 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554712, ip=192.168.3.6, actor_id=f724992af97278a8bfe6afb601000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7ee8d0d70700>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,891\tERROR actor_manager.py:507 -- Ray error, taking actor 9 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554713, ip=192.168.3.6, actor_id=ac9bde688625551e7eb9537401000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f4d0234d6a0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,894\tERROR actor_manager.py:507 -- Ray error, taking actor 10 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554714, ip=192.168.3.6, actor_id=df229fe140f491ebeec4202201000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f176ebe96d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,897\tERROR actor_manager.py:507 -- Ray error, taking actor 11 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554715, ip=192.168.3.6, actor_id=94b93c07fdaecd69323651f301000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f557406b730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,900\tERROR actor_manager.py:507 -- Ray error, taking actor 12 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554716, ip=192.168.3.6, actor_id=05714b62c61e7adb5937562e01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fc82b0f6730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,903\tERROR actor_manager.py:507 -- Ray error, taking actor 13 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554717, ip=192.168.3.6, actor_id=725a28258f1b91b5f5a3d9fa01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f0c05be8760>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,907\tERROR actor_manager.py:507 -- Ray error, taking actor 14 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554718, ip=192.168.3.6, actor_id=5514696fa77fe2f3e44d3ed601000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fbaf27b9700>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,913\tERROR actor_manager.py:507 -- Ray error, taking actor 15 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554719, ip=192.168.3.6, actor_id=a2b4fc8693a5bf52bf42079b01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fe681173730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,916\tERROR actor_manager.py:507 -- Ray error, taking actor 16 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554720, ip=192.168.3.6, actor_id=e34826feb2c9ed8eb6b4b2fa01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7eebb6595730>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,919\tERROR actor_manager.py:507 -- Ray error, taking actor 17 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554721, ip=192.168.3.6, actor_id=7266d7b34bd7d9b25efd9b6a01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f8c29ecc670>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,922\tERROR actor_manager.py:507 -- Ray error, taking actor 18 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554722, ip=192.168.3.6, actor_id=3b266589ba4b26a3fd37cf7f01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7eeb9cacb700>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,925\tERROR actor_manager.py:507 -- Ray error, taking actor 19 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554723, ip=192.168.3.6, actor_id=6bb94e72064ef982e99a6f8401000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f2c42be96d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,928\tERROR actor_manager.py:507 -- Ray error, taking actor 20 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554724, ip=192.168.3.6, actor_id=e64fbdbac5111efc6258e0b201000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f49581b9760>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,931\tERROR actor_manager.py:507 -- Ray error, taking actor 21 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554725, ip=192.168.3.6, actor_id=8aa7d6b8039688cb6dccdacd01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f5c7cf0b6d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,934\tERROR actor_manager.py:507 -- Ray error, taking actor 22 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554726, ip=192.168.3.6, actor_id=5ea299d22c1088c737da7b4b01000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7eff8222c790>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,937\tERROR actor_manager.py:507 -- Ray error, taking actor 23 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554727, ip=192.168.3.6, actor_id=6811dabbe9ecbcd60089208901000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7fafc63256a0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n",
      "2023-10-10 10:05:37,940\tERROR actor_manager.py:507 -- Ray error, taking actor 24 out of service. The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=2554728, ip=192.168.3.6, actor_id=c7c098296217b1162cb7559101000000, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x7f56762ec6d0>)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 738, in __init__\n",
      "    self._update_policy_map(policy_dict=self.policy_dict)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 1985, in _update_policy_map\n",
      "    self._build_policy_map(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py\", line 2097, in _build_policy_map\n",
      "    new_policy = create_policy_for_framework(\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py\", line 142, in create_policy_for_framework\n",
      "    return policy_class(observation_space, action_space, merged_config)\n",
      "  File \"/home/yuan/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py\", line 54, in __init__\n",
      "    max_seq_len=config[\"model\"][\"max_seq_len\"],\n",
      "KeyError: 'max_seq_len'\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'max_seq_len'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m/home/yuan/ResMan/testApi.ipynb 单元格 49\u001b[0m line \u001b[0;36m5\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=50'>51</a>\u001b[0m     config\u001b[39m.\u001b[39mreuse_actors \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=51'>52</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m config\u001b[39m.\u001b[39mbuild()\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=52'>53</a>\u001b[0m tuner \u001b[39m=\u001b[39m Tuner\u001b[39m.\u001b[39mrestore(\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=53'>54</a>\u001b[0m     path\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m/home/yuan/ray_results/PPO6_20_lstm/PPO_VJS_00d92_00000_0_2023-10-10_09-26-14\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=54'>55</a>\u001b[0m     trainable\u001b[39m=\u001b[39mget_trainable(\u001b[39m20\u001b[39;49m,\u001b[39m6\u001b[39;49m,\u001b[39m24\u001b[39;49m),\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=55'>56</a>\u001b[0m     resume_unfinished\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=56'>57</a>\u001b[0m     resume_errored\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=57'>58</a>\u001b[0m     restart_errored\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=58'>59</a>\u001b[0m )\n",
      "\u001b[1;32m/home/yuan/ResMan/testApi.ipynb 单元格 49\u001b[0m line \u001b[0;36m5\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=47'>48</a>\u001b[0m \u001b[39m# print(config.to_dict())\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=48'>49</a>\u001b[0m \u001b[39m# Build a Algorithm object from the config and run one training iteration.\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=49'>50</a>\u001b[0m \u001b[39m# algo = config.build(env=env_name)\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=50'>51</a>\u001b[0m config\u001b[39m.\u001b[39mreuse_actors \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bgtx6180/home/yuan/ResMan/testApi.ipynb#X66sdnNjb2RlLXJlbW90ZQ%3D%3D?line=51'>52</a>\u001b[0m \u001b[39mreturn\u001b[39;00m config\u001b[39m.\u001b[39;49mbuild()\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/algorithm_config.py:1071\u001b[0m, in \u001b[0;36mAlgorithmConfig.build\u001b[0;34m(self, env, logger_creator, use_copy)\u001b[0m\n\u001b[1;32m   1068\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39malgo_class, \u001b[39mstr\u001b[39m):\n\u001b[1;32m   1069\u001b[0m     algo_class \u001b[39m=\u001b[39m get_trainable_cls(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39malgo_class)\n\u001b[0;32m-> 1071\u001b[0m \u001b[39mreturn\u001b[39;00m algo_class(\n\u001b[1;32m   1072\u001b[0m     config\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m \u001b[39mif\u001b[39;49;00m \u001b[39mnot\u001b[39;49;00m use_copy \u001b[39melse\u001b[39;49;00m copy\u001b[39m.\u001b[39;49mdeepcopy(\u001b[39mself\u001b[39;49m),\n\u001b[1;32m   1073\u001b[0m     logger_creator\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlogger_creator,\n\u001b[1;32m   1074\u001b[0m )\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/algorithm.py:475\u001b[0m, in \u001b[0;36mAlgorithm.__init__\u001b[0;34m(self, config, env, logger_creator, **kwargs)\u001b[0m\n\u001b[1;32m    457\u001b[0m \u001b[39m# Initialize common evaluation_metrics to nan, before they become\u001b[39;00m\n\u001b[1;32m    458\u001b[0m \u001b[39m# available. We want to make sure the metrics are always present\u001b[39;00m\n\u001b[1;32m    459\u001b[0m \u001b[39m# (although their values may be nan), so that Tune does not complain\u001b[39;00m\n\u001b[1;32m    460\u001b[0m \u001b[39m# when we use these as stopping criteria.\u001b[39;00m\n\u001b[1;32m    461\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mevaluation_metrics \u001b[39m=\u001b[39m {\n\u001b[1;32m    462\u001b[0m     \u001b[39m# TODO: Don't dump sampler results into top-level.\u001b[39;00m\n\u001b[1;32m    463\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mevaluation\u001b[39m\u001b[39m\"\u001b[39m: {\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    472\u001b[0m     },\n\u001b[1;32m    473\u001b[0m }\n\u001b[0;32m--> 475\u001b[0m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49m\u001b[39m__init__\u001b[39;49m(\n\u001b[1;32m    476\u001b[0m     config\u001b[39m=\u001b[39;49mconfig,\n\u001b[1;32m    477\u001b[0m     logger_creator\u001b[39m=\u001b[39;49mlogger_creator,\n\u001b[1;32m    478\u001b[0m     \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs,\n\u001b[1;32m    479\u001b[0m )\n\u001b[1;32m    481\u001b[0m \u001b[39m# Check, whether `training_iteration` is still a tune.Trainable property\u001b[39;00m\n\u001b[1;32m    482\u001b[0m \u001b[39m# and has not been overridden by the user in the attempt to implement the\u001b[39;00m\n\u001b[1;32m    483\u001b[0m \u001b[39m# algos logic (this should be done now inside `training_step`).\u001b[39;00m\n\u001b[1;32m    484\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/tune/trainable/trainable.py:170\u001b[0m, in \u001b[0;36mTrainable.__init__\u001b[0;34m(self, config, logger_creator, remote_checkpoint_dir, custom_syncer, sync_timeout, sync_config)\u001b[0m\n\u001b[1;32m    168\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_start_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime()\n\u001b[1;32m    169\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_local_ip \u001b[39m=\u001b[39m ray\u001b[39m.\u001b[39mutil\u001b[39m.\u001b[39mget_node_ip_address()\n\u001b[0;32m--> 170\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup(copy\u001b[39m.\u001b[39;49mdeepcopy(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconfig))\n\u001b[1;32m    171\u001b[0m setup_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime() \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_start_time\n\u001b[1;32m    172\u001b[0m \u001b[39mif\u001b[39;00m setup_time \u001b[39m>\u001b[39m SETUP_TIME_THRESHOLD:\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/algorithm.py:601\u001b[0m, in \u001b[0;36mAlgorithm.setup\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m    594\u001b[0m \u001b[39m# Only if user did not override `_init()`:\u001b[39;00m\n\u001b[1;32m    595\u001b[0m \u001b[39mif\u001b[39;00m _init \u001b[39mis\u001b[39;00m \u001b[39mFalse\u001b[39;00m:\n\u001b[1;32m    596\u001b[0m     \u001b[39m# - Create rollout workers here automatically.\u001b[39;00m\n\u001b[1;32m    597\u001b[0m     \u001b[39m# - Run the execution plan to create the local iterator to `next()`\u001b[39;00m\n\u001b[1;32m    598\u001b[0m     \u001b[39m#   in each training iteration.\u001b[39;00m\n\u001b[1;32m    599\u001b[0m     \u001b[39m# This matches the behavior of using `build_trainer()`, which\u001b[39;00m\n\u001b[1;32m    600\u001b[0m     \u001b[39m# has been deprecated.\u001b[39;00m\n\u001b[0;32m--> 601\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mworkers \u001b[39m=\u001b[39m WorkerSet(\n\u001b[1;32m    602\u001b[0m         env_creator\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49menv_creator,\n\u001b[1;32m    603\u001b[0m         validate_env\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvalidate_env,\n\u001b[1;32m    604\u001b[0m         default_policy_class\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_default_policy_class(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconfig),\n\u001b[1;32m    605\u001b[0m         config\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconfig,\n\u001b[1;32m    606\u001b[0m         num_workers\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconfig\u001b[39m.\u001b[39;49mnum_rollout_workers,\n\u001b[1;32m    607\u001b[0m         local_worker\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[1;32m    608\u001b[0m         logdir\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlogdir,\n\u001b[1;32m    609\u001b[0m     )\n\u001b[1;32m    611\u001b[0m     \u001b[39m# TODO (avnishn): Remove the execution plan API by q1 2023\u001b[39;00m\n\u001b[1;32m    612\u001b[0m     \u001b[39m# Function defining one single training iteration's behavior.\u001b[39;00m\n\u001b[1;32m    613\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39m_disable_execution_plan_api:\n\u001b[1;32m    614\u001b[0m         \u001b[39m# Ensure remote workers are initially in sync with the local worker.\u001b[39;00m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/worker_set.py:194\u001b[0m, in \u001b[0;36mWorkerSet.__init__\u001b[0;34m(self, env_creator, validate_env, default_policy_class, config, num_workers, local_worker, logdir, _setup, policy_class, trainer_config)\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[39mexcept\u001b[39;00m RayActorError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    182\u001b[0m     \u001b[39m# In case of an actor (remote worker) init failure, the remote worker\u001b[39;00m\n\u001b[1;32m    183\u001b[0m     \u001b[39m# may still exist and will be accessible, however, e.g. calling\u001b[39;00m\n\u001b[1;32m    184\u001b[0m     \u001b[39m# its `sample.remote()` would result in strange \"property not found\"\u001b[39;00m\n\u001b[1;32m    185\u001b[0m     \u001b[39m# errors.\u001b[39;00m\n\u001b[1;32m    186\u001b[0m     \u001b[39mif\u001b[39;00m e\u001b[39m.\u001b[39mactor_init_failed:\n\u001b[1;32m    187\u001b[0m         \u001b[39m# Raise the original error here that the RolloutWorker raised\u001b[39;00m\n\u001b[1;32m    188\u001b[0m         \u001b[39m# during its construction process. This is to enforce transparency\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    192\u001b[0m         \u001b[39m# - e.args[0].args[2]: The original Exception (e.g. a ValueError due\u001b[39;00m\n\u001b[1;32m    193\u001b[0m         \u001b[39m# to a config mismatch) thrown inside the actor.\u001b[39;00m\n\u001b[0;32m--> 194\u001b[0m         \u001b[39mraise\u001b[39;00m e\u001b[39m.\u001b[39margs[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39margs[\u001b[39m2\u001b[39m]\n\u001b[1;32m    195\u001b[0m     \u001b[39m# In any other case, raise the RayActorError as-is.\u001b[39;00m\n\u001b[1;32m    196\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    197\u001b[0m         \u001b[39mraise\u001b[39;00m e\n",
      "File \u001b[0;32mpython/ray/_raylet.pyx:1431\u001b[0m, in \u001b[0;36mray._raylet.execute_task\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpython/ray/_raylet.pyx:1510\u001b[0m, in \u001b[0;36mray._raylet.execute_task\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpython/ray/_raylet.pyx:1434\u001b[0m, in \u001b[0;36mray._raylet.execute_task\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpython/ray/_raylet.pyx:1438\u001b[0m, in \u001b[0;36mray._raylet.execute_task\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32mpython/ray/_raylet.pyx:1378\u001b[0m, in \u001b[0;36mray._raylet.execute_task.function_executor\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/_private/function_manager.py:724\u001b[0m, in \u001b[0;36mactor_method_executor\u001b[0;34m()\u001b[0m\n\u001b[1;32m    722\u001b[0m     \u001b[39mreturn\u001b[39;00m method(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    723\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 724\u001b[0m     \u001b[39mreturn\u001b[39;00m method(__ray_actor, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/util/tracing/tracing_helper.py:464\u001b[0m, in \u001b[0;36m_resume_span\u001b[0;34m()\u001b[0m\n\u001b[1;32m    462\u001b[0m \u001b[39m# If tracing feature flag is not on, perform a no-op\u001b[39;00m\n\u001b[1;32m    463\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m _is_tracing_enabled() \u001b[39mor\u001b[39;00m _ray_trace_ctx \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 464\u001b[0m     \u001b[39mreturn\u001b[39;00m method(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m_args, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m_kwargs)\n\u001b[1;32m    466\u001b[0m tracer: _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mTracer \u001b[39m=\u001b[39m _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mget_tracer(\n\u001b[1;32m    467\u001b[0m     \u001b[39m__name__\u001b[39m\n\u001b[1;32m    468\u001b[0m )\n\u001b[1;32m    470\u001b[0m \u001b[39m# Retrieves the context from the _ray_trace_ctx dictionary we\u001b[39;00m\n\u001b[1;32m    471\u001b[0m \u001b[39m# injected.\u001b[39;00m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py:738\u001b[0m, in \u001b[0;36m__init__\u001b[0;34m()\u001b[0m\n\u001b[1;32m    736\u001b[0m \u001b[39m# if RLModule API is enabled, marl_module_spec holds the specs of the RLModules\u001b[39;00m\n\u001b[1;32m    737\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmarl_module_spec \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 738\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_update_policy_map(policy_dict\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpolicy_dict)\n\u001b[1;32m    740\u001b[0m \u001b[39m# Update Policy's view requirements from Model, only if Policy directly\u001b[39;00m\n\u001b[1;32m    741\u001b[0m \u001b[39m# inherited from base `Policy` class. At this point here, the Policy\u001b[39;00m\n\u001b[1;32m    742\u001b[0m \u001b[39m# must have it's Model (if any) defined and ready to output an initial\u001b[39;00m\n\u001b[1;32m    743\u001b[0m \u001b[39m# state.\u001b[39;00m\n\u001b[1;32m    744\u001b[0m \u001b[39mfor\u001b[39;00m pol \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpolicy_map\u001b[39m.\u001b[39mvalues():\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/util/tracing/tracing_helper.py:464\u001b[0m, in \u001b[0;36m_resume_span\u001b[0;34m()\u001b[0m\n\u001b[1;32m    462\u001b[0m \u001b[39m# If tracing feature flag is not on, perform a no-op\u001b[39;00m\n\u001b[1;32m    463\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m _is_tracing_enabled() \u001b[39mor\u001b[39;00m _ray_trace_ctx \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 464\u001b[0m     \u001b[39mreturn\u001b[39;00m method(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m_args, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m_kwargs)\n\u001b[1;32m    466\u001b[0m tracer: _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mTracer \u001b[39m=\u001b[39m _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mget_tracer(\n\u001b[1;32m    467\u001b[0m     \u001b[39m__name__\u001b[39m\n\u001b[1;32m    468\u001b[0m )\n\u001b[1;32m    470\u001b[0m \u001b[39m# Retrieves the context from the _ray_trace_ctx dictionary we\u001b[39;00m\n\u001b[1;32m    471\u001b[0m \u001b[39m# injected.\u001b[39;00m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py:1985\u001b[0m, in \u001b[0;36m_update_policy_map\u001b[0;34m()\u001b[0m\n\u001b[1;32m   1980\u001b[0m     updated_policy_dict \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_update_policy_dict_with_marl_module(\n\u001b[1;32m   1981\u001b[0m         updated_policy_dict\n\u001b[1;32m   1982\u001b[0m     )\n\u001b[1;32m   1984\u001b[0m \u001b[39m# Builds the self.policy_map dict\u001b[39;00m\n\u001b[0;32m-> 1985\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_policy_map(\n\u001b[1;32m   1986\u001b[0m     policy_dict\u001b[39m=\u001b[39mupdated_policy_dict,\n\u001b[1;32m   1987\u001b[0m     policy\u001b[39m=\u001b[39mpolicy,\n\u001b[1;32m   1988\u001b[0m     policy_states\u001b[39m=\u001b[39mpolicy_states,\n\u001b[1;32m   1989\u001b[0m )\n\u001b[1;32m   1991\u001b[0m \u001b[39m# Initialize the filter dict\u001b[39;00m\n\u001b[1;32m   1992\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_update_filter_dict(updated_policy_dict)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/util/tracing/tracing_helper.py:464\u001b[0m, in \u001b[0;36m_resume_span\u001b[0;34m()\u001b[0m\n\u001b[1;32m    462\u001b[0m \u001b[39m# If tracing feature flag is not on, perform a no-op\u001b[39;00m\n\u001b[1;32m    463\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m _is_tracing_enabled() \u001b[39mor\u001b[39;00m _ray_trace_ctx \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 464\u001b[0m     \u001b[39mreturn\u001b[39;00m method(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39m_args, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m_kwargs)\n\u001b[1;32m    466\u001b[0m tracer: _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mTracer \u001b[39m=\u001b[39m _opentelemetry\u001b[39m.\u001b[39mtrace\u001b[39m.\u001b[39mget_tracer(\n\u001b[1;32m    467\u001b[0m     \u001b[39m__name__\u001b[39m\n\u001b[1;32m    468\u001b[0m )\n\u001b[1;32m    470\u001b[0m \u001b[39m# Retrieves the context from the _ray_trace_ctx dictionary we\u001b[39;00m\n\u001b[1;32m    471\u001b[0m \u001b[39m# injected.\u001b[39;00m\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/evaluation/rollout_worker.py:2097\u001b[0m, in \u001b[0;36m_build_policy_map\u001b[0;34m()\u001b[0m\n\u001b[1;32m   2093\u001b[0m \u001b[39mfor\u001b[39;00m name, policy_spec \u001b[39min\u001b[39;00m \u001b[39msorted\u001b[39m(policy_dict\u001b[39m.\u001b[39mitems()):\n\u001b[1;32m   2094\u001b[0m \n\u001b[1;32m   2095\u001b[0m     \u001b[39m# Create the actual policy object.\u001b[39;00m\n\u001b[1;32m   2096\u001b[0m     \u001b[39mif\u001b[39;00m policy \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 2097\u001b[0m         new_policy \u001b[39m=\u001b[39m create_policy_for_framework(\n\u001b[1;32m   2098\u001b[0m             policy_id\u001b[39m=\u001b[39mname,\n\u001b[1;32m   2099\u001b[0m             policy_class\u001b[39m=\u001b[39mget_tf_eager_cls_if_necessary(\n\u001b[1;32m   2100\u001b[0m                 policy_spec\u001b[39m.\u001b[39mpolicy_class, policy_spec\u001b[39m.\u001b[39mconfig\n\u001b[1;32m   2101\u001b[0m             ),\n\u001b[1;32m   2102\u001b[0m             merged_config\u001b[39m=\u001b[39mpolicy_spec\u001b[39m.\u001b[39mconfig,\n\u001b[1;32m   2103\u001b[0m             observation_space\u001b[39m=\u001b[39mpolicy_spec\u001b[39m.\u001b[39mobservation_space,\n\u001b[1;32m   2104\u001b[0m             action_space\u001b[39m=\u001b[39mpolicy_spec\u001b[39m.\u001b[39maction_space,\n\u001b[1;32m   2105\u001b[0m             worker_index\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mworker_index,\n\u001b[1;32m   2106\u001b[0m             seed\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mseed,\n\u001b[1;32m   2107\u001b[0m         )\n\u001b[1;32m   2108\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   2109\u001b[0m         new_policy \u001b[39m=\u001b[39m policy\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/utils/policy.py:142\u001b[0m, in \u001b[0;36mcreate_policy_for_framework\u001b[0;34m()\u001b[0m\n\u001b[1;32m    139\u001b[0m             \u001b[39mreturn\u001b[39;00m policy_class(observation_space, action_space, merged_config)\n\u001b[1;32m    140\u001b[0m \u001b[39m# Non-tf: No graph, no session.\u001b[39;00m\n\u001b[1;32m    141\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 142\u001b[0m     \u001b[39mreturn\u001b[39;00m policy_class(observation_space, action_space, merged_config)\n",
      "File \u001b[0;32m~/ResMan/man/lib/python3.9/site-packages/ray/rllib/algorithms/ppo/ppo_torch_policy.py:54\u001b[0m, in \u001b[0;36m__init__\u001b[0;34m()\u001b[0m\n\u001b[1;32m     46\u001b[0m config \u001b[39m=\u001b[39m \u001b[39mdict\u001b[39m(ray\u001b[39m.\u001b[39mrllib\u001b[39m.\u001b[39malgorithms\u001b[39m.\u001b[39mppo\u001b[39m.\u001b[39mppo\u001b[39m.\u001b[39mPPOConfig()\u001b[39m.\u001b[39mto_dict(), \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mconfig)\n\u001b[1;32m     47\u001b[0m validate_config(config)\n\u001b[1;32m     49\u001b[0m TorchPolicyV2\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\n\u001b[1;32m     50\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m     51\u001b[0m     observation_space,\n\u001b[1;32m     52\u001b[0m     action_space,\n\u001b[1;32m     53\u001b[0m     config,\n\u001b[0;32m---> 54\u001b[0m     max_seq_len\u001b[39m=\u001b[39mconfig[\u001b[39m\"\u001b[39m\u001b[39mmodel\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mmax_seq_len\u001b[39m\u001b[39m\"\u001b[39m],\n\u001b[1;32m     55\u001b[0m )\n\u001b[1;32m     57\u001b[0m ValueNetworkMixin\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, config)\n\u001b[1;32m     58\u001b[0m LearningRateSchedule\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, config[\u001b[39m\"\u001b[39m\u001b[39mlr\u001b[39m\u001b[39m\"\u001b[39m], config[\u001b[39m\"\u001b[39m\u001b[39mlr_schedule\u001b[39m\u001b[39m\"\u001b[39m])\n",
      "\u001b[0;31mKeyError\u001b[0m: 'max_seq_len'"
     ]
    }
   ],
   "source": [
    "from ray.tune import Tuner\n",
    "from ray import tune\n",
    "from ray.rllib.algorithms.ppo import PPOConfig\n",
    "from ray.rllib.policy.policy import PolicySpec\n",
    "from ray.tune import register_env\n",
    "import rllib_setup\n",
    "def get_trainable(jobs, machine, rollout_workers=10):\n",
    "    env_name = \"VJS\"\n",
    "    alg_name = \"PPO\"\n",
    "    register_env(\n",
    "        env_name,\n",
    "        lambda config: rllib_setup.get_env_continuous(jobs, machine),\n",
    "    )\n",
    "    test_env = rllib_setup.get_env_continuous(jobs, machine)\n",
    "\n",
    "    def policies(agent_ids):\n",
    "        obs_space = test_env.observation_space\n",
    "        act_space = test_env.action_space\n",
    "        return {\n",
    "            str(i): (\n",
    "                None,\n",
    "                obs_space,\n",
    "                act_space,\n",
    "                {}\n",
    "                # config=config.overrides(agent_id=int(i[8:])),\n",
    "            )\n",
    "            for i in agent_ids\n",
    "        }\n",
    "\n",
    "    config = (\n",
    "        PPOConfig()\n",
    "        .rollouts(num_rollout_workers=rollout_workers)\n",
    "        .training(vf_clip_param=1080)\n",
    "        .resources(num_gpus=2)\n",
    "        .multi_agent(\n",
    "            policies=policies(test_env._agent_ids),\n",
    "            policy_mapping_fn=lambda agent_id, episode, **kwargs: str(agent_id),\n",
    "        )\n",
    "        .environment(env=env_name, disable_env_checking=True)\n",
    "        .evaluation(evaluation_interval=10)\n",
    "    )\n",
    "    config.model= {\n",
    "                \"fcnet_hiddens\": [384, 256, 128],\n",
    "                \"fcnet_activation\": \"relu\",\n",
    "                \"use_lstm\": True,\n",
    "                \"lstm_cell_size\": 256,\n",
    "            }\n",
    "    # print(config.to_dict())\n",
    "    # Build a Algorithm object from the config and run one training iteration.\n",
    "    # algo = config.build(env=env_name)\n",
    "    config.reuse_actors = True\n",
    "    return config.build()\n",
    "tuner = Tuner.restore(\n",
    "    path=\"/home/yuan/ray_results/PPO6_20_lstm/PPO_VJS_00d92_00000_0_2023-10-10_09-26-14\",\n",
    "    trainable=get_trainable(20,6,24),\n",
    "    resume_unfinished=True,\n",
    "    resume_errored=True,\n",
    "    restart_errored=False,\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "man",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d652fbd7feb28eaf2ff622dc57929f599600a55ad62e1efe6c9f098053cf5e75"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}