Sample transitions from replay buffer with _forward steps_.
        That is, instead of getting a transition s_t -> s_t+1 with reward r,
        get a transition s_t -> s_t+n with sum of intermediate rewards.

        Used in a variant of Deep Q-Learning
19521
def sample_forward_transitions(self, batch_size, batch_info, forward_steps: int,
                                   discount_factor: float) -> Transitions:
        
        indexes = self.backend.sample_batch_transitions(batch_size, forward_steps=forward_steps)
        transition_tensors = self.backend.get_transitions_forward_steps(
            indexes, forward_steps=forward_steps, discount_factor=discount_factor
        )

        return Trajectories(
            num_steps=batch_size,
            num_envs=self.backend.num_envs,
            environment_information=None,
            transition_tensors={k: torch.from_numpy(v) for k, v in transition_tensors.items()},
            rollout_tensors={},
            extra_data={
                'forward_steps': forward_steps
            }
        ).to_transitions()
1186620
private int getPreviousStep(int currentStep) {
        int prevStep = -1;

        // we find the next available step.
        for (int i = currentStep - 1; i >= STEP_LOGIN; i--) {
            if(isStepEnabled(i)) {
                prevStep = i;
                break;
            }
        }

        return prevStep;
    }
887604
def increment_step_and_update_last_reward(self):
        
        if len(self.stats['Environment/Cumulative Reward']) > 0:
            mean_reward = np.mean(self.stats['Environment/Cumulative Reward'])
            self.policy.update_reward(mean_reward)
        self.policy.increment_step()
        self.step = self.policy.get_current_step()
438099
def forward(self, agent_qs, states):
        
        bs = agent_qs.size(0)
        states = states.reshape(-1, self.state_dim)
        agent_qs = agent_qs.view(-1, 1, self.n_agents)
        # First layer
        w1 = th.abs(self.hyper_w_1(states))
        b1 = self.hyper_b_1(states)
        w1 = w1.view(-1, self.n_agents, self.embed_dim)
        b1 = b1.view(-1, 1, self.embed_dim)
        hidden = F.elu(th.bmm(agent_qs, w1) + b1)
        # Second layer
        w_final = th.abs(self.hyper_w_final(states))
        w_final = w_final.view(-1, self.embed_dim, 1)
        # State-dependent bias
        v = self.V(states).view(-1, 1, 1)
        # Compute final output
        y = th.bmm(hidden, w_final) + v
        # Reshape and return
        q_tot = y.view(bs, -1, 1)
        return q_tot
587476
def sample_batch_transitions(self, batch_size, forward_steps=1):
        
        results = []

        for i in range(self.num_envs):
            results.append(self.sample_frame_single_env(batch_size, forward_steps=forward_steps))

        return np.stack(results, axis=-1)
327849
def fixed_step_return(reward, value, length, discount, window):
  """"""
  timestep = tf.range(reward.shape[1].value)
  mask = tf.cast(timestep[None, :] < length[:, None], tf.float32)
  return_ = tf.zeros_like(reward)
  for _ in range(window):
    return_ += reward
    reward = discount * tf.concat(
        [reward[:, 1:], tf.zeros_like(reward[:, -1:])], 1)
  return_ += discount ** window * tf.concat(
      [value[:, window:], tf.zeros_like(value[:, -window:])], 1)
  return tf.check_numerics(tf.stop_gradient(mask * return_), 'return')
737409
def rollout(self, batch_info: BatchInfo, model: RlModel, number_of_steps: int) -> Rollout:
        
        assert not model.is_recurrent, "Replay env roller does not support recurrent models"

        accumulator = TensorAccumulator()
        episode_information = []  # List of dictionaries with episode information

        for step_idx in range(number_of_steps):
            step = model.step(self.last_observation)

            replay_extra_information = {}

            accumulator.add('observations', self.last_observation_cpu)

            # Add step to the tensor accumulator
            for name, tensor in step.items():
                tensor_cpu = tensor.cpu()
                accumulator.add(name, tensor_cpu)

                if name != 'actions':
                    replay_extra_information[name] = tensor_cpu.numpy()

            actions_numpy = step['actions'].detach().cpu().numpy()
            new_obs, new_rewards, new_dones, new_infos = self.environment.step(actions_numpy)

            # Store rollout in the experience replay buffer
            self.replay_buffer.store_transition(
                frame=self.last_observation_cpu.numpy(),
                action=actions_numpy,
                reward=new_rewards,
                done=new_dones,
                extra_info=replay_extra_information
            )

            # Done is flagged true when the episode has ended AND the frame we see is already a first frame from the
            # next episode

            dones_tensor = torch.from_numpy(new_dones.astype(np.float32)).clone()
            accumulator.add('dones', dones_tensor)

            self.last_observation_cpu = torch.from_numpy(new_obs).clone()
            self.last_observation = self.last_observation_cpu.to(self.device)
            accumulator.add('rewards', torch.from_numpy(new_rewards.astype(np.float32)).clone())

            episode_information.append(new_infos)

        accumulated_tensors = accumulator.result()

        return Trajectories(
            num_steps=accumulated_tensors['observations'].size(0),
            num_envs=accumulated_tensors['observations'].size(1),
            environment_information=episode_information,
            transition_tensors=accumulated_tensors,
            rollout_tensors={
                'final_values': model.value(self.last_observation).cpu()
            }
        )
262195
def step(self, exploration):
        
        old_s = self._current_ob
        if self.rng.rand() <= exploration:
            act = self.rng.choice(range(self.num_actions))
        else:
            history = self.recent_state()
            history.append(old_s)
            history = np.stack(history, axis=-1)  # state_shape + (Hist,)

            # assume batched network
            history = np.expand_dims(history, axis=0)
            q_values = self.predictor(history)[0][0]  # this is the bottleneck
            act = np.argmax(q_values)

        self._current_ob, reward, isOver, info = self.player.step(act)
        self._current_game_score.feed(reward)
        self._current_episode.append(Experience(old_s, act, reward, isOver))

        if isOver:
            flush_experience = True
            if 'ale.lives' in info:  # if running Atari, do something special
                if info['ale.lives'] != 0:
                    # only record score and flush experience
                    # when a whole game is over (not when an episode is over)
                    flush_experience = False
            self.player.reset()

            if flush_experience:
                self.total_scores.append(self._current_game_score.sum)
                self._current_game_score.reset()

                # Ensure that the whole episode of experience is continuous in the replay buffer
                with self.memory.writer_lock:
                    for exp in self._current_episode:
                        self.memory.append(exp)
                self._current_episode.clear()
199624
def sample_transitions(self, batch_size, batch_info, discount_factor=None) -> Transitions:
        
        indexes = self.backend.sample_batch_transitions(batch_size)
        transition_tensors = self.backend.get_transitions(indexes)

        return Trajectories(
            num_steps=batch_size,
            num_envs=self.backend.num_envs,
            environment_information=None,
            transition_tensors={k: torch.from_numpy(v) for k, v in transition_tensors.items()},
            rollout_tensors={}
        ).to_transitions()
771592
def learn(self, initial_state_key, limit=1000, game_n=1):
        '''
        
            
        '''
        end_flag = False
        state_key_list = [None] * len(self.q_learning_list)
        action_key_list = [None] * len(self.q_learning_list)
        next_action_key_list = [None] * len(self.q_learning_list)
        for game in range(game_n):
            state_key = initial_state_key
            self.t = 1
            while self.t <= limit:
                for i in range(len(self.q_learning_list)):
                    state_key_list[i] = state_key
                    if game + 1 == game_n:
                        self.state_key_list.append(tuple(i, state_key_list))
                    self.q_learning_list[i].t = self.t
                    next_action_list = self.q_learning_list[i].extract_possible_actions(tuple(i, state_key_list))
                    if len(next_action_list):
                        action_key = self.q_learning_list[i].select_action(
                            state_key=tuple(i, state_key_list),
                            next_action_list=next_action_list
                        )
                        action_key_list[i] = action_key
                        reward_value = self.q_learning_list[i].observe_reward_value(
                            tuple(i, state_key_list), 
                            tuple(i, action_key_list)
                        )

                        # Check.
                        if self.q_learning_list[i].check_the_end_flag(tuple(i, state_key_list)) is True:
                            end_flag = True

                        # Max-Q-Value in next action time.
                        next_next_action_list = self.q_learning_list[i].extract_possible_actions(
                            tuple(i, action_key_list)
                        )
                        if len(next_next_action_list):
                            next_action_key = self.q_learning_list[i].predict_next_action(
                                tuple(i, action_key_list), 
                                next_next_action_list
                            )
                            next_action_key_list[i] = next_action_key
                            next_max_q = self.q_learning_list[i].extract_q_df(
                                tuple(i, action_key_list), 
                                next_action_key
                            )

                            # Update Q-Value.
                            self.q_learning_list[i].update_q(
                                state_key=tuple(i, state_key_list),
                                action_key=tuple(i, action_key_list),
                                reward_value=reward_value,
                                next_max_q=next_max_q
                            )

                            # Update State.
                            state_key = self.q_learning_list[i].update_state(
                                state_key=tuple(i, state_key_list),
                                action_key=tuple(i, action_key_list)
                            )
                            state_key_list[i] = state_key

                    # Epsode.
                    self.t += 1
                    self.q_learning_list[i].t = self.t
                    if end_flag is True:
                        break
105042
def get_transitions_forward_steps(self, indexes, forward_steps, discount_factor):
        
        return self.deque.get_transitions_forward_steps(indexes, forward_steps, discount_factor)
687486
def get_transition(self, frame_idx, env_idx):
        
        past_frame, future_frame = self.get_frame_with_future(frame_idx, env_idx)

        data_dict = {
            'observations': past_frame,
            'observations_next': future_frame,
            'actions': self.action_buffer[frame_idx, env_idx],
            'rewards': self.reward_buffer[frame_idx, env_idx],
            'dones': self.dones_buffer[frame_idx, env_idx],
        }

        for name in self.extra_data:
            data_dict[name] = self.extra_data[name][frame_idx, env_idx]

        return data_dict
1288921
def collect_random_trajectory(env, timesteps=1000):
    

    obs = env.reset()
    dof = env.dof

    for t in range(timesteps):
        action = 0.5 * np.random.randn(dof)
        obs, reward, done, info = env.step(action)
        env.render()
        if t % 100 == 0:
            print(t)
759666
def _generate_time_steps(self, trajectory_list):
    """"""
    for single_trajectory in trajectory_list:
      assert isinstance(single_trajectory, trajectory.Trajectory)

      # Skip writing trajectories that have only a single time-step -- this
      # could just be a repeated reset.

      if single_trajectory.num_time_steps <= 1:
        continue

      for index, time_step in enumerate(single_trajectory.time_steps):

        # The first time-step doesn't have reward/processed_reward, if so, just
        # setting it to 0.0 / 0 should be OK.
        raw_reward = time_step.raw_reward
        if not raw_reward:
          raw_reward = 0.0

        processed_reward = time_step.processed_reward
        if not processed_reward:
          processed_reward = 0

        action = time_step.action
        if action is None:
          # The last time-step doesn't have action, and this action shouldn't be
          # used, gym's spaces have a `sample` function, so let's just sample an
          # action and use that.
          action = self.action_space.sample()
        action = gym_spaces_utils.gym_space_encode(self.action_space, action)

        if six.PY3:
          # py3 complains that, to_example cannot handle np.int64 !

          action_dtype = self.action_space.dtype
          if action_dtype in [np.int64, np.int32]:
            action = list(map(int, action))
          elif action_dtype in [np.float64, np.float32]:
            action = list(map(float, action))

          # same with processed_reward.
          processed_reward = int(processed_reward)

        assert time_step.observation is not None

        yield {
            TIMESTEP_FIELD: [index],
            ACTION_FIELD:
                action,
            # to_example errors on np.float32
            RAW_REWARD_FIELD: [float(raw_reward)],
            PROCESSED_REWARD_FIELD: [processed_reward],
            # to_example doesn't know bools
            DONE_FIELD: [int(time_step.done)],
            OBSERVATION_FIELD:
                gym_spaces_utils.gym_space_encode(self.observation_space,
                                                  time_step.observation),
        }
944122
def step(self, loss, optimizer, scheduler, update=True):
        
        loss.backward()
        if update:
            if self.grad_clip != float('inf'):
                clip_grad_norm_(self.model.parameters(), self.grad_clip)
            scheduler.step()
            optimizer.step()
            self.model.zero_grad()
1212293
def rl_loop():
  """"""

  state = State()

  if FLAGS.checkpoint_dir:
    # Start from a partially trained model.
    initialize_from_checkpoint(state)
  else:
    # Play the first round of selfplay games with a fake model that returns
    # random noise. We do this instead of playing multiple games using a single
    # model bootstrapped with random noise to avoid any initial bias.
    wait(selfplay(state, 'bootstrap'))

    # Train a real model from the random selfplay games.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    # Select the newly trained model as the best.
    state.best_model_name = state.train_model_name
    state.gen_num += 1

    # Run selfplay using the new model.
    wait(selfplay(state))

  # Now start the full training loop.
  while state.iter_num <= FLAGS.iterations:
    # Build holdout glob before incrementing the iteration number because we
    # want to run validation on the previous generation.
    holdout_glob = os.path.join(fsdb.holdout_dir(), '%06d-*' % state.iter_num,
                                '*')

    # Train on shuffled game data from recent selfplay rounds.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    if FLAGS.parallel_post_train:
      # Run eval, validation & selfplay in parallel.
      model_win_rate, _, _ = wait([
          evaluate_trained_model(state),
          validate(state, holdout_glob),
          selfplay(state)])
    else:
      # Run eval, validation & selfplay sequentially.
      model_win_rate = wait(evaluate_trained_model(state))
      wait(validate(state, holdout_glob))
      wait(selfplay(state))

    if model_win_rate >= FLAGS.gating_win_rate:
      # Promote the trained model to the best model and increment the generation
      # number.
      state.best_model_name = state.train_model_name
      state.gen_num += 1
571946
def doTimestep(self, locationSDR, transitionSDR, featureSDR,
                 egocentricLocation, learn):
    

    for monitor in self.monitors.values():
      monitor.beforeTimestep(locationSDR, transitionSDR, featureSDR,
                             egocentricLocation, learn)

    params = {
      "newLocation": locationSDR,
      "deltaLocation": transitionSDR,
      "featureLocationInput": self.inputLayer.getActiveCells(),
      "featureLocationGrowthCandidates": self.inputLayer.getPredictedActiveCells(),
      "learn": learn,
    }
    self.locationLayer.compute(**params)
    for monitor in self.monitors.values():
      monitor.afterLocationCompute(**params)

    params = {
      "activeColumns": featureSDR,
      "basalInput": self.locationLayer.getActiveCells(),
      "apicalInput": self.objectLayer.getActiveCells(),
    }
    self.inputLayer.compute(**params)
    for monitor in self.monitors.values():
      monitor.afterInputCompute(**params)

    params = {
      "feedforwardInput": self.inputLayer.getActiveCells(),
      "feedforwardGrowthCandidates": self.inputLayer.getPredictedActiveCells(),
      "learn": learn,
    }
    self.objectLayer.compute(**params)
    for monitor in self.monitors.values():
      monitor.afterObjectCompute(**params)