<jupyter_start><jupyter_text>This is a jupyter notebook demonstrating usage of [garage](https://github.com/rlworkgroup/garage) in a jupyter notebook.In particular, it demonstrates the example `trpo_gym_tf_cartpole.py` file already available in [garage/examples/tf/](https://github.com/rlworkgroup/garage/blob/master/examples/tf/)  Install pre-requisites<jupyter_code>%%shell

echo "abcd" > mujoco_fake_key


git clone --depth 1 https://github.com/rlworkgroup/garage/

cd garage
bash scripts/setup_colab.sh --mjkey ../mujoco_fake_key --no-modify-bashrc > /dev/null
raise Exception("Please restart your runtime so that the installed dependencies for 'garage' can be loaded, and then resume running the notebook")<jupyter_output><empty_output><jupyter_text>---------  Prepare for the training<jupyter_code># The contents of this cell and the one after are mostly copied from garage/examples/...
# Note that these need to be run twice if for the first time on a colab.research.google.com instance
# 1st time is to create the "personal config from template"
# 2nd time is the charm

from garage.np.baselines import LinearFeatureBaseline
from garage.envs import normalize
# from garage.envs.box2d import CartpoleEnv
from garage.experiment import run_experiment
from garage.tf.algos import TRPO
from garage.tf.envs import TfEnv
#from garage.tf.policies import GaussianMLPPolicy
from garage.tf.policies import CategoricalMLPPolicy

import gym

from garage.experiment import LocalRunner
from garage.logger import logger, StdOutput

# garage version of CartPole environment, has Discrete action space instead of Box
#env = TfEnv(normalize(CartpoleEnv()))
#policy = GaussianMLPPolicy(
#    name="policy", env_spec=env.spec, hidden_sizes=(32, 32))

# gym version of CartPole. Check note above
# env = TfEnv(normalize(gym.make("CartPole-v0")))

# garage updated method of getting env
env = TfEnv(env_name='CartPole-v1')
policy = CategoricalMLPPolicy(
    name="policy", env_spec=env.spec, hidden_sizes=(32, 32))

# create baseline
baseline = LinearFeatureBaseline(env_spec=env.spec)

# specify an algorithm
algo = TRPO(
    env_spec=env.spec,
    policy=policy,
    baseline=baseline,
    
    # Use these settings for garage version of env
    # max_path_length=100,
    # n_itr=100,
    
    # Use these for gym version
    max_path_length=200,
    n_itr=20,

    discount=0.99,
    max_kl_step=0.01
    )<jupyter_output><empty_output><jupyter_text>Train the algorithm<jupyter_code># start a tensorflow session so that we can keep it open after training and use the trained network to see it performing
import tensorflow as tf
sess = tf.InteractiveSession()

# initialize
sess.run(tf.compat.v1.global_variables_initializer())
# log to stdout
logger.add_output(StdOutput())
# train the algo

runner = LocalRunner()

runner.setup(algo=algo, env=env)

# use n_epochs = 100 for practical example, n_epochs = 10 for quick demo, n_epochs = 1 for smoke testing
runner.train(n_epochs=10, batch_size=10000, plot=False)<jupyter_output>2019-04-01 05:46:28 | epoch #0 | Obtaining samples...
2019-04-01 05:46:28 | epoch #0 | Obtaining samples for iteration 0...<jupyter_text>Visualize a video of the algorithm playing<jupyter_code>%%shell

# Prepare display for seeing a video of the policy in action in the jupyter notebook
# Note that this doesn't require a runtime restart
# https://stackoverflow.com/a/51183488/4126114

apt-get install python-opengl ffmpeg xvfb
pip install pyvirtualdisplay
# Virtual display
from pyvirtualdisplay import Display

virtual_display = Display(visible=0, size=(1400, 900))
virtual_display.start()
# bugfix?
# Set an "id" field since missing for some reason (uncovered in monitor wrapper below)
env.spec.id = 1
# wrap the gym environment for recording a video of the policy performance
# https://kyso.io/eoin/openai-gym-jupyter?utm_campaign=News&utm_medium=Community&utm_source=DataCamp.com
import gym
from gym import wrappers

env = wrappers.Monitor(env, "./gym-results", force=True)
obs = env.reset()
for i in range(1000):
    #action = env.action_space.sample()
    action, _ = policy.get_action(obs)
    obs, reward, done, info = env.step(action)
    if done: break

print("done at step %i"%i)
env.close()
# Display the video in the jupyter notebook
# Click the play button below to watch the video
import io
import base64
from IPython.display import HTML

video = io.open('./gym-results/openaigym.video.%s.video000000.mp4' % env.file_infix, 'r+b').read()
encoded = base64.b64encode(video)

vid_ascii = encoded.decode('ascii')
HTML(data='''
    <video width="360" height="auto" alt="test" controls><source src="data:video/mp4;base64,{0}" type="video/mp4" /></video>'''
.format(vid_ascii))<jupyter_output><empty_output>