<jupyter_start><jupyter_text>Setup rendering dependencies for Google Colaboratory.<jupyter_code>!pip install gym pyvirtualdisplay > /dev/null 2>&1
!apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1<jupyter_output><empty_output><jupyter_text>Install d3rlpy!<jupyter_code>!pip install d3rlpy<jupyter_output>Collecting d3rlpy
  Downloading d3rlpy-1.1.0-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
[K     |████████████████████████████████| 1.2 MB 5.2 MB/s 
[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (1.11.0+cu113)
Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (1.0.2)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (4.2.0)
Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (7.1.2)
Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (1.4.1)
Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (1.3.0)
Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from d3rlpy) (3.1.0)
Collecting tensorboardX
  Downloading tensorboardX-2.5-py2.py3-none-any.whl (125 kB)
[K     |███████████████[...]<jupyter_text>Setup cartpole dataset.<jupyter_code>from d3rlpy.datasets import get_cartpole

# get CartPole dataset
dataset, env = get_cartpole()<jupyter_output>Downloading cartpole.pkl into d3rlpy_data/cartpole_replay_v1.1.0.h5...<jupyter_text>Setup data-driven deep reinforcement learning algorithm.<jupyter_code>from d3rlpy.algos import DiscreteCQL
from d3rlpy.metrics.scorer import discounted_sum_of_advantage_scorer
from d3rlpy.metrics.scorer import evaluate_on_environment
from d3rlpy.metrics.scorer import td_error_scorer
from d3rlpy.metrics.scorer import average_value_estimation_scorer
from sklearn.model_selection import train_test_split

# setup CQL algorithm
cql = DiscreteCQL(use_gpu=False)

# split train and test episodes
train_episodes, test_episodes = train_test_split(dataset, test_size=0.2)

# start training
cql.fit(train_episodes,
        eval_episodes=test_episodes,
        n_epochs=1,
        scorers={
            'environment': evaluate_on_environment(env), # evaluate with CartPol-v0 environment
            'advantage': discounted_sum_of_advantage_scorer, # smaller is better
            'td_error': td_error_scorer, # smaller is better
            'value_scale': average_value_estimation_scorer # smaller is better
        })<jupyter_output>2022-05-06 06:02.00 [debug    ] RoundIterator is selected.
2022-05-06 06:02.00 [info     ] Directory is created at d3rlpy_logs/DiscreteCQL_20220506060200
2022-05-06 06:02.00 [debug    ] Building models...
2022-05-06 06:02.00 [debug    ] Models have been built.
2022-05-06 06:02.00 [info     ] Parameters are saved to d3rlpy_logs/DiscreteCQL_20220506060200/params.json params={'action_scaler': None, 'alpha': 1.0, 'batch_size': 32, 'encoder_factory': {'type': 'default', 'params': {'activation': 'relu', 'use_batch_norm': False, 'dropout_rate': None}}, 'gamma': 0.99, 'generated_maxlen': 100000, 'learning_rate': 6.25e-05, 'n_critics': 1, 'n_frames': 1, 'n_steps': 1, 'optim_factory': {'optim_cls': 'Adam', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'q_func_factory': {'type': 'mean', 'params': {'share_encoder': False}}, 'real_ratio': 1.0, 'reward_scaler': None, 'scaler': None, 'target_update_interval': 8000, 'use_gpu': None, 'algorithm': 'DiscreteCQL', 'observation[...]<jupyter_text>Setup rendering utilities for Google Colaboratory.<jupyter_code>import glob
import io
import base64

from gym.wrappers import Monitor
from IPython.display import HTML
from IPython import display as ipythondisplay
from pyvirtualdisplay import Display

# start virtual display
display = Display(visible=0, size=(1400, 900))
display.start()

# play recorded video
def show_video():
    mp4list = glob.glob('video/*.mp4')
    if len(mp4list) > 0:
        mp4 = mp4list[0]
        video = io.open(mp4, 'r+b').read()
        encoded = base64.b64encode(video)
        ipythondisplay.display(HTML(data='''
            <video alt="test" autoplay loop controls style="height: 400px;">
                <source src="data:video/mp4;base64,{0}" type="video/mp4" />
            </video>'''.format(encoded.decode('ascii'))))
    else: 
        print("Could not find video")<jupyter_output><empty_output><jupyter_text>Record video!<jupyter_code># wrap Monitor wrapper
env = Monitor(env, './video', force=True)

# evaluate
evaluate_on_environment(env)(cql)<jupyter_output><empty_output><jupyter_text>Let's see how it works!<jupyter_code>show_video()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup