<jupyter_start><jupyter_text>Car racinghttps://gym.openai.com/envs/CarRacing-v0/  Install dependanciesDacă ceva nu merge, renunțați la `> /dev/null` pentru a vă asigura că instalarea a mers blană.<jupyter_code>!pip install gym pyvirtualdisplay > /dev/null 2>&1
!apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1
!apt-get update > /dev/null 2>&1
!apt-get install cmake > /dev/null 2>&1
!pip install --upgrade setuptools 2>&1
!pip install ez_setup > /dev/null 2>&1
!pip install gym[atari] > /dev/null 2>&1
!pip install box2d-py > /dev/null 2>&1<jupyter_output>Requirement already up-to-date: setuptools in /usr/local/lib/python3.6/dist-packages (41.0.1)<jupyter_text>Imports and Helper functions<jupyter_code>import gym
from gym import logger as gymlogger
from gym.wrappers import Monitor
gymlogger.set_level(40) #error only
import tensorflow as tf
import numpy as np
import random
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
import math
import glob
import io
import base64
from IPython.display import HTML

from IPython import display as ipythondisplay

import gym
from gym import logger as gymlogger
from gym.wrappers import Monitor
gymlogger.set_level(40) #error only
import tensorflow as tf
import numpy as np
from collections import namedtuple
from itertools import count
from PIL import Image
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torchvision.transforms as T
import random
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
import math
import glob
import io
import base64
from IPython.display import HTML
from IPython import display as ipythondisplay
from pyvirtualdisplay import Display
from pyvirtualdisplay import Display
display = Display(visible=0, size=(1400, 900))
display.start()
# <NAME>: https://star-ai.github.io/Rendering-OpenAi-Gym-in-Colaboratory/

def show_video():
  mp4list = glob.glob('video/*.mp4')
  if len(mp4list) > 0:
    mp4 = mp4list[0]
    video = io.open(mp4, 'r+b').read()
    encoded = base64.b64encode(video)
    ipythondisplay.display(HTML(data='''<video alt="test" autoplay 
                loop controls style="height: 400px;">
                <source src="data:video/mp4;base64,{0}" type="video/mp4" />
             </video>'''.format(encoded.decode('ascii'))))
  else: 
    print("Could not find video")
    

def wrap_env(env):
  env = Monitor(env, './video', force=True)
  return env<jupyter_output><empty_output><jupyter_text>CarRacing - vizualizare<jupyter_code>env = wrap_env(gym.make("CarRacing-v0"))
observation = env.reset()

while True:
    env.render()
    action = env.action_space.sample() 
    observation, reward, done, info = env.step(action) 
           
    if done: 
        break;
env.close()

show_video()<jupyter_output>Track generation: 1143..1433 -> 290-tiles track<jupyter_text>CarRacing - fără vizualizare<jupyter_code>env = wrap_env(gym.make("CarRacing-v0"))
env.action_space = [[1, 0, 0], [-1, 0, 0], [0, 1, 0], [0, 0, 0.8]]
NR_ACTIONS = len(env.action_space)
env.reset()

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
Transition = namedtuple('Transition',
                        ('state', 'action', 'next_state', 'reward'))


class ReplayMemory(object):

    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, *args):
        """Saves a transition."""
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)
class DQN(nn.Module):

    def __init__(self, h, w, outputs):
        super(DQN, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)
        self.bn3 = nn.BatchNorm2d(32)

        # Number of Linear input connections depends on output of conv2d layers
        # and therefore the input image size, so compute it.
        def conv2d_size_out(size, kernel_size = 5, stride = 2):
            return (size - (kernel_size - 1) - 1) // stride  + 1
        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))
        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))
        linear_input_size = convw * convh * 32
        self.head = nn.Linear(linear_input_size, outputs)

    # Called with either one element to determine next action, or a batch
    # during optimization. Returns tensor([[left0exp,right0exp]...]).
    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))
        return self.head(x.view(x.size(0), -1))
resize = T.Compose([T.ToPILImage(),
                    T.Resize((96, 96), interpolation=Image.CUBIC), # TODO adjust sizes
                    T.ToTensor()])

def get_screen():
    screen = env.render(mode='rgb_array').transpose((2, 0, 1))
    
    # Resize, and add a batch dimension (BCHW)
    return resize(screen).unsqueeze(0).to(device)
# Set up parameters, select_action and optimize_model functions
BATCH_SIZE = 128
GAMMA = 0.999
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200
TARGET_UPDATE = 10

# Get screen size so that we can initialize layers correctly based on shape
# returned from AI gym. Typical dimensions at this point are close to 3x40x90
# which is the result of a clamped and down-scaled render buffer in get_screen()
env.reset()
init_screen = get_screen()
_, _, screen_height, screen_width = init_screen.shape
print(init_screen.shape)

# Get number of actions from gym action space
#n_actions = env.action_space.n # old
n_actions = NR_ACTIONS # new

policy_net = DQN(screen_height, screen_width, n_actions).to(device)
target_net = DQN(screen_height, screen_width, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()

optimizer = optim.RMSprop(policy_net.parameters())
memory = ReplayMemory(10000)


steps_done = 0


def select_action(state):
    global steps_done
    sample = random.random()
    eps_threshold = EPS_END + (EPS_START - EPS_END) * \
        math.exp(-1. * steps_done / EPS_DECAY)
    steps_done += 1
    if sample > eps_threshold:
        with torch.no_grad():
            # t.max(1) will return largest column value of each row.
            # second column on max result is index of where max element was
            # found, so we pick action with the larger expected reward.
            return policy_net(state).max(1)[1].view(1, 1)
    else:
        return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)


episode_durations = []
def optimize_model():
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
    # detailed explanation). This converts batch-array of Transitions
    # to Transition of batch-arrays.
    batch = Transition(*zip(*transitions))

    # Compute a mask of non-final states and concatenate the batch elements
    # (a final state would've been the one after which simulation ended)
    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
                                          batch.next_state)), device=device, dtype=torch.uint8)
    non_final_next_states = torch.cat([s for s in batch.next_state
                                                if s is not None])
    state_batch = torch.cat(batch.state)
    action_batch = torch.cat(batch.action)
    reward_batch = torch.cat(batch.reward)

    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
    # columns of actions taken. These are the actions which would've been taken
    # for each batch state according to policy_net
    state_action_values = policy_net(state_batch).gather(1, action_batch)

    # Compute V(s_{t+1}) for all next states.
    # Expected values of actions for non_final_next_states are computed based
    # on the "older" target_net; selecting their best reward with max(1)[0].
    # This is merged based on the mask, such that we'll have either the expected
    # state value or 0 in case the state was final.
    next_state_values = torch.zeros(BATCH_SIZE, device=device)
    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()
    # Compute the expected Q values
    expected_state_action_values = (next_state_values * GAMMA) + reward_batch

    # Compute Huber loss
    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))

    # Optimize the model
    optimizer.zero_grad()
    loss.backward()
    for param in policy_net.parameters():
        param.grad.data.clamp_(-1, 1)
    optimizer.step()
# NEW TESTING VERSION
import time

num_episodes = 15
for i_episode in range(num_episodes):
  # Initialize the environment and state
  env.reset()
  last_screen = get_screen()
  current_screen = get_screen()
  state = current_screen - last_screen
  start = time.time()
  for t in count():
      action = select_action(state)
      #observation, reward, done, info = env.step(action.item()) # old - from tutorial
      observation, reward, done, info = env.step(env.action_space[action.item()]) # new
      reward = torch.tensor([reward], device=device).float()

      # Observe new state
      last_screen = current_screen
      current_screen = get_screen()
      if not done:
          next_state = current_screen - last_screen
      else:
          next_state = None
          
      # Store the transition in memory
      memory.push(state, action, next_state, reward)

      # Move to the next state
      state = next_state
      
      # Perform one step of the optimization (on the target network)
      optimize_model()
      if done:
          episode_durations.append(t + 1)
          if len(episode_durations) == 20:
            print("Average ep. length (last 20 eps.): ", np.mean(episode_durations))
            episode_durations.clear()
            
          print('Took', time.time() - start, 'seconds for 1 episode')
          start = time.time()
          break
  # Update the target network, copying all weights and biases in DQN
  if i_episode % TARGET_UPDATE == 0:
      target_net.load_state_dict(policy_net.state_dict())

print('Complete')

env.close()

# show_video()
class DDQN(nn.Module):

    def __init__(self, h, w, outputs):
        super(DDQN, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)
        self.bn1 = nn.BatchNorm2d(16)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)
        self.bn3 = nn.BatchNorm2d(32)

        # Number of Linear input connections depends on output of conv2d layers
        # and therefore the input image size, so compute it.
        def conv2d_size_out(size, kernel_size = 5, stride = 2):
            return (size - (kernel_size - 1) - 1) // stride  + 1
        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))
        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))
        linear_input_size = convw * convh * 32
        self.head1 = nn.Linear(linear_input_size, outputs)
        linear_input_size_inter = convw * convh
        self.head21 = nn.Linear(linear_input_size, linear_input_size_inter)
        self.head22 = nn.Linear(linear_input_size_inter, outputs)

    # Called with either one element to determine next action, or a batch
    # during optimization. Returns tensor([[left0exp,right0exp]...]).
    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))
        a = self.head1(x.view(x.size(0), -1))
        b = self.head21(x.view(x.size(0), -1))
        c = self.head22(b)
        return a + c
resize = T.Compose([T.ToPILImage(),
                    T.Resize((96, 96), interpolation=Image.CUBIC), # TODO adjust sizes
                    T.ToTensor()])

def get_screen():
    screen = env.render(mode='rgb_array').transpose((2, 0, 1))
    
    # Resize, and add a batch dimension (BCHW)
    return resize(screen).unsqueeze(0).to(device)
# Set up parameters, select_action and optimize_model functions
BATCH_SIZE = 128
GAMMA = 0.999
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200
TARGET_UPDATE = 10

# Get screen size so that we can initialize layers correctly based on shape
# returned from AI gym. Typical dimensions at this point are close to 3x40x90
# which is the result of a clamped and down-scaled render buffer in get_screen()
env.reset()
init_screen = get_screen()
_, _, screen_height, screen_width = init_screen.shape
print(init_screen.shape)

# Get number of actions from gym action space
#n_actions = env.action_space.n # old
n_actions = NR_ACTIONS # new

policy_net = DDQN(screen_height, screen_width, n_actions).to(device)
target_net = DDQN(screen_height, screen_width, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()

optimizer = optim.RMSprop(policy_net.parameters())
memory = ReplayMemory(10000)


steps_done = 0


def select_action(state):
    global steps_done
    sample = random.random()
    eps_threshold = EPS_END + (EPS_START - EPS_END) * \
        math.exp(-1. * steps_done / EPS_DECAY)
    steps_done += 1
    if sample > eps_threshold:
        with torch.no_grad():
            # t.max(1) will return largest column value of each row.
            # second column on max result is index of where max element was
            # found, so we pick action with the larger expected reward.
            return policy_net(state).max(1)[1].view(1, 1)
    else:
        return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)


episode_durations = []
def optimize_model():
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
    # detailed explanation). This converts batch-array of Transitions
    # to Transition of batch-arrays.
    batch = Transition(*zip(*transitions))

    # Compute a mask of non-final states and concatenate the batch elements
    # (a final state would've been the one after which simulation ended)
    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
                                          batch.next_state)), device=device, dtype=torch.uint8)
    non_final_next_states = torch.cat([s for s in batch.next_state
                                                if s is not None])
    state_batch = torch.cat(batch.state)
    action_batch = torch.cat(batch.action)
    reward_batch = torch.cat(batch.reward)

    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
    # columns of actions taken. These are the actions which would've been taken
    # for each batch state according to policy_net
    state_action_values = policy_net(state_batch).gather(1, action_batch)

    # Compute V(s_{t+1}) for all next states.
    # Expected values of actions for non_final_next_states are computed based
    # on the "older" target_net; selecting their best reward with max(1)[0].
    # This is merged based on the mask, such that we'll have either the expected
    # state value or 0 in case the state was final.
    next_state_values = torch.zeros(BATCH_SIZE, device=device)
    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()
    # Compute the expected Q values
    expected_state_action_values = (next_state_values * GAMMA) + reward_batch

    # Compute Huber loss
    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))

    # Optimize the model
    optimizer.zero_grad()
    loss.backward()
    for param in policy_net.parameters():
        param.grad.data.clamp_(-1, 1)
    optimizer.step()
# NEW TESTING VERSION
import time

num_episodes = 15
for i_episode in range(num_episodes):
  # Initialize the environment and state
  env.reset()
  last_screen = get_screen()
  current_screen = get_screen()
  state = current_screen - last_screen
  start = time.time()
  for t in count():
      action = select_action(state)
      #observation, reward, done, info = env.step(action.item()) # old - from tutorial
      observation, reward, done, info = env.step(env.action_space[action.item()]) # new
      reward = torch.tensor([reward], device=device).float()

      # Observe new state
      last_screen = current_screen
      current_screen = get_screen()
      if not done:
          next_state = current_screen - last_screen
      else:
          next_state = None
          
      # Store the transition in memory
      memory.push(state, action, next_state, reward)

      # Move to the next state
      state = next_state
      
      # Perform one step of the optimization (on the target network)
      optimize_model()
      if done:
          episode_durations.append(t + 1)
          if len(episode_durations) == 20:
            print("Average ep. length (last 20 eps.): ", np.mean(episode_durations))
            episode_durations.clear()
            
          print('Took', time.time() - start, 'seconds for 1 episode')
          start = time.time()
          break
  # Update the target network, copying all weights and biases in DDQN
  if i_episode % TARGET_UPDATE == 0:
      target_net.load_state_dict(policy_net.state_dict())

print('Complete')

env.close()

# show_video()<jupyter_output>Track generation: 1099..1378 -> 279-tiles track
Took 49.952375173568726 seconds for 1 episode
Track generation: 1100..1379 -> 279-tiles track
Took 51.98528051376343 seconds for 1 episode
Track generation: 1275..1598 -> 323-tiles track
Took 54.00662970542908 seconds for 1 episode
Track generation: 1087..1370 -> 283-tiles track
Took 52.510228395462036 seconds for 1 episode
Track generation: 1211..1518 -> 307-tiles track
Took 53.164055824279785 seconds for 1 episode
Track generation: 1160..1454 -> 294-tiles track
Took 52.34375 seconds for 1 episode
Track generation: 1108..1394 -> 286-tiles track
Took 53.41212344169617 seconds for 1 episode
Track generation: 1241..1564 -> 323-tiles track
Took 55.53093338012695 seconds for 1 episode
Track generation: 1252..1569 -> 317-tiles track
Took 53.23687934875488 seconds for 1 episode
Track generation: 1131..1418 -> 287-tiles track
Took 52.6376428604126 seconds for 1 episode
Track generation: 1353..1695 -> 342-tiles track
Took 54.39814209938049 second[...]