03_minerl_navigate_agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp agents.navigate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic MineRL Agent\n",
    "> Run an agent for a navigation task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "import time\n",
    "\n",
    "import gym\n",
    "import minerl  # noqa\n",
    "\n",
    "# OpenCog\n",
    "from opencog.logger import log\n",
    "from opencog.pln import *\n",
    "from opencog.type_constructors import *\n",
    "from opencog.utilities import set_default_atomspace\n",
    "\n",
    "# ROCCA\n",
    "from rocca.envs.wrappers import MineRLWrapper\n",
    "from rocca.envs.wrappers.minerl_wrapper import minerl_single_action\n",
    "from rocca.agents import OpencogAgent\n",
    "from rocca.agents.utils import *\n",
    "\n",
    "from rocca.utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is pretty much `OpencogAgent` but with the `step` method fixed to use the `minerl_single_action` helper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "class NavigateAgent(OpencogAgent):\n",
    "    def __init__(self, env, action_space, p_goal, n_goal, log_level=\"info\"):\n",
    "        OpencogAgent.__init__(self, env, action_space, p_goal, n_goal, log_level)\n",
    "\n",
    "    def control_cycle(self):\n",
    "        \"\"\"Run one step of observation, decision and env update\"\"\"\n",
    "        agent_log.debug(\"atomese_obs = {}\".format(self.observation))\n",
    "        obs_record = [\n",
    "            self.record(o, self.cycle_count, tv=TRUE_TV) for o in self.observation\n",
    "        ]\n",
    "        agent_log.debug(\"obs_record = {}\".format(obs_record))\n",
    "\n",
    "        # Make the goal for that iteration\n",
    "        goal = self.make_goal()\n",
    "        agent_log.debug(\"goal = {}\".format(goal))\n",
    "\n",
    "        # Plan, i.e. come up with cognitive schematics as plans.  Here the\n",
    "        # goal expiry is 2, i.e. must be fulfilled set for the next two iterations.\n",
    "        cogscms = self.plan(goal, self.expiry)\n",
    "        agent_log.debug(\"cogscms = {}\".format(cogscms))\n",
    "\n",
    "        # Deduce the action distribution\n",
    "        mxmdl = self.deduce(cogscms)\n",
    "        agent_log.debug(\"mxmdl = {}\".format(mxmdl_to_str(mxmdl)))\n",
    "\n",
    "        # Select the next action\n",
    "        action, pblty = self.decide(mxmdl)\n",
    "        agent_log.debug(\n",
    "            \"action with probability of success = {}\".format(\n",
    "                act_pblt_to_str((action, pblty))\n",
    "            )\n",
    "        )\n",
    "\n",
    "        # Timestamp the action that is about to be executed\n",
    "        action_record = self.record(action, self.cycle_count, tv=TRUE_TV)\n",
    "        agent_log.debug(\"action_record = {}\".format(action_record))\n",
    "        agent_log.debug(\"action = {}\".format(action))\n",
    "\n",
    "        # Increment the counter for that action and log it\n",
    "        self.action_counter[action] += 1\n",
    "        agent_log.debug(\"action_counter = {}\".format(self.action_counter))\n",
    "\n",
    "        # Increase the step count and run the next step of the environment\n",
    "        self.cycle_count += 1\n",
    "        # TODO gather environment info.\n",
    "        reward, self.observation, done = self.env.step(\n",
    "            minerl_single_action(self.env, action)\n",
    "        )\n",
    "\n",
    "        self.accumulated_reward += float(reward.out[1].name)\n",
    "        agent_log.debug(\"observation = {}\".format(self.observation))\n",
    "        agent_log.debug(\"reward = {}\".format(reward))\n",
    "        agent_log.debug(\"accumulated reward = {}\".format(self.accumulated_reward))\n",
    "\n",
    "        reward_record = self.record(reward, self.cycle_count, tv=TRUE_TV)\n",
    "        agent_log.debug(\"reward_record = {}\".format(reward_record))\n",
    "\n",
    "        if done:\n",
    "            return False\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment\n",
    "\n",
    "For this experiment we use the basic _MineRLNavigateDense-v0_ environment of MineRL. The task of the agent is to touch a diamond block placed somewhere in the vicinity. This is a _Dense_ version of the environment, which means the agent receives rewards all the time, based on whether the distance to the target decreases or increases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"MineRLNavigateDense-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "atomspace = AtomSpace()\n",
    "set_default_atomspace(atomspace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrapped_env = MineRLWrapper(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Goal\n",
    "pgoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"100\"))\n",
    "ngoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"0\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_space = {\n",
    "    ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"back\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"back\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"left\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"left\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"right\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"right\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"dirt\")),\n",
    "    ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"none\")),\n",
    "    ExecutionLink(SchemaNode(\"camera\"), ListLink(NumberNode(\"2.5\"), NumberNode(\"0.0\"))),\n",
    "    ExecutionLink(\n",
    "        SchemaNode(\"camera\"), ListLink(NumberNode(\"0.0\"), NumberNode(\"-1.5\"))\n",
    "    ),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = NavigateAgent(wrapped_env, action_space, pgoal, ngoal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "tb_writer = SummaryWriter(comment=\"-minerl-navigate\")\n",
    "\n",
    "epochs = 1  # Number of epochs (learning / interacting episodes)\n",
    "epoch_len = 200\n",
    "\n",
    "for i in range(epochs):\n",
    "    wrapped_env.restart()\n",
    "    agent.reset_action_counter()\n",
    "    accreward = agent.accumulated_reward  # Keep track of the reward before\n",
    "\n",
    "    # Learning phase: discover patterns to make more informed decisions\n",
    "    log_msg(agent_log, f\"Learning phase started. ({i + 1}/{epochs})\")\n",
    "    agent.learn()\n",
    "\n",
    "    # Run agent to accumulate percepta\n",
    "    log_msg(agent_log, f\"Interaction phase started. ({i + 1}/{epochs})\")\n",
    "    for j in range(epoch_len):\n",
    "        done = agent.control_cycle()\n",
    "        # wrapped_env.render() uncomment to see the rendered env\n",
    "        time.sleep(0.01)\n",
    "        log.info(\"cycle_count = {}\".format(agent.cycle_count))\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "    new_reward = agent.accumulated_reward - accreward\n",
    "    tb_writer.add_scalar(\"train/accumulated_reward\", new_reward, agent.cycle_count)\n",
    "    log_msg(\n",
    "        agent_log, \"Accumulated reward during {}th epoch = {}\".format(i + 1, new_reward)\n",
    "    )\n",
    "    log_msg(\n",
    "        agent_log,\n",
    "        \"Action counter during {}th epoch:\\n{}\".format(i + 1, agent.action_counter),\n",
    "    )  # TODO: make the action counter look good\n",
    "\n",
    "log_msg(agent_log, f\"The average total reward over {epochs} trials (training): {agent.accumulated_reward / epochs}.\")"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}