Multi-Agent Learning.

        Override.
        
        Args:
            initial_state_key:  Initial state.
            limit:              Limit of the number of learning.
            game_n:             The number of games.
771592
def learn(self, initial_state_key, limit=1000, game_n=1):
        '''
        
            
        '''
        end_flag = False
        state_key_list = [None] * len(self.q_learning_list)
        action_key_list = [None] * len(self.q_learning_list)
        next_action_key_list = [None] * len(self.q_learning_list)
        for game in range(game_n):
            state_key = initial_state_key
            self.t = 1
            while self.t <= limit:
                for i in range(len(self.q_learning_list)):
                    state_key_list[i] = state_key
                    if game + 1 == game_n:
                        self.state_key_list.append(tuple(i, state_key_list))
                    self.q_learning_list[i].t = self.t
                    next_action_list = self.q_learning_list[i].extract_possible_actions(tuple(i, state_key_list))
                    if len(next_action_list):
                        action_key = self.q_learning_list[i].select_action(
                            state_key=tuple(i, state_key_list),
                            next_action_list=next_action_list
                        )
                        action_key_list[i] = action_key
                        reward_value = self.q_learning_list[i].observe_reward_value(
                            tuple(i, state_key_list), 
                            tuple(i, action_key_list)
                        )

                        # Check.
                        if self.q_learning_list[i].check_the_end_flag(tuple(i, state_key_list)) is True:
                            end_flag = True

                        # Max-Q-Value in next action time.
                        next_next_action_list = self.q_learning_list[i].extract_possible_actions(
                            tuple(i, action_key_list)
                        )
                        if len(next_next_action_list):
                            next_action_key = self.q_learning_list[i].predict_next_action(
                                tuple(i, action_key_list), 
                                next_next_action_list
                            )
                            next_action_key_list[i] = next_action_key
                            next_max_q = self.q_learning_list[i].extract_q_df(
                                tuple(i, action_key_list), 
                                next_action_key
                            )

                            # Update Q-Value.
                            self.q_learning_list[i].update_q(
                                state_key=tuple(i, state_key_list),
                                action_key=tuple(i, action_key_list),
                                reward_value=reward_value,
                                next_max_q=next_max_q
                            )

                            # Update State.
                            state_key = self.q_learning_list[i].update_state(
                                state_key=tuple(i, state_key_list),
                                action_key=tuple(i, action_key_list)
                            )
                            state_key_list[i] = state_key

                    # Epsode.
                    self.t += 1
                    self.q_learning_list[i].t = self.t
                    if end_flag is True:
                        break
648817
def load_optimizer_states(self, fname):
        
        assert self._updater is not None, "Cannot load states for distributed training"
        self._updater.set_states(open(fname, 'rb').read())
685275
def get_states(self, n):
    
    return self.states[len(self.new_states):len(self.new_states) + n]
1331840
public void rnnSetPreviousState(String layerName, Map<String, INDArray> state) {
        Layer l = verticesMap.get(layerName).getLayer();
        if(l instanceof org.deeplearning4j.nn.layers.wrapper.BaseWrapperLayer){
            l = ((org.deeplearning4j.nn.layers.wrapper.BaseWrapperLayer)l).getUnderlying();
        }
        if (l == null || !(l instanceof RecurrentLayer)) {
            throw new UnsupportedOperationException(
                    "Layer \"" + layerName + "\" is not a recurrent layer. Cannot set state");
        }
        ((RecurrentLayer) l).rnnSetPreviousState(state);
    }
648399
def _oneInteraction(self):
        
        self.stepid += 1

        logger.info("Entering simulation period %d." % self.stepid)

        # Apply branches outages.
        if self.branchOutages is not None:
            self.doOutages()

        # Initialise the market.
        self.market.reset()

        # Get an action from each agent and perform it.
        for task, agent in zip(self.tasks, self.agents):
#            if self.do_optimisation[agent]:
#                raise Exception("When using a black-box learning algorithm, "
#                                "only full episodes can be done.")

#            if not task.isFinished():
            observation = task.getObservation()
            agent.integrateObservation(observation)

            action = agent.getAction()
            task.performAction(action)

        # Clear the market.
        self.market.run()

        # Reward each agent appropriately.
        for task, agent in zip(self.tasks, self.agents):
#            if not task.isFinished():
            reward = task.getReward()
            agent.giveReward(reward)

        # Scale loads.
        c = self._pcycle.next()
        for bus in self.market.case.buses:
            bus.p_demand = self.pdemand[bus] * c

        logger.info("")
589301
def on_train_begin(self, n_epochs:int, epoch:int, **kwargs:Any)->None:
        ""
        res = {'epoch':self.start_epoch} if self.start_epoch is not None else None
        self.start_epoch = ifnone(self.start_epoch, epoch)
        self.tot_epochs = ifnone(self.tot_epochs, n_epochs)
        n = len(self.learn.data.train_dl) * self.tot_epochs
        a1 = int(n * self.pct_start)
        a2 = n-a1
        self.phases = ((a1, annealing_cos), (a2, annealing_cos))
        low_lr = self.lr_max/self.div_factor
        self.lr_scheds = self.steps((low_lr, self.lr_max), (self.lr_max, self.lr_max/self.final_div))
        self.mom_scheds = self.steps(self.moms, (self.moms[1], self.moms[0]))
        self.opt = self.learn.opt
        self.opt.lr,self.opt.mom = self.lr_scheds[0].start,self.mom_scheds[0].start
        self.idx_s = 0
        return res
1212293
def rl_loop():
  """"""

  state = State()

  if FLAGS.checkpoint_dir:
    # Start from a partially trained model.
    initialize_from_checkpoint(state)
  else:
    # Play the first round of selfplay games with a fake model that returns
    # random noise. We do this instead of playing multiple games using a single
    # model bootstrapped with random noise to avoid any initial bias.
    wait(selfplay(state, 'bootstrap'))

    # Train a real model from the random selfplay games.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    # Select the newly trained model as the best.
    state.best_model_name = state.train_model_name
    state.gen_num += 1

    # Run selfplay using the new model.
    wait(selfplay(state))

  # Now start the full training loop.
  while state.iter_num <= FLAGS.iterations:
    # Build holdout glob before incrementing the iteration number because we
    # want to run validation on the previous generation.
    holdout_glob = os.path.join(fsdb.holdout_dir(), '%06d-*' % state.iter_num,
                                '*')

    # Train on shuffled game data from recent selfplay rounds.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    if FLAGS.parallel_post_train:
      # Run eval, validation & selfplay in parallel.
      model_win_rate, _, _ = wait([
          evaluate_trained_model(state),
          validate(state, holdout_glob),
          selfplay(state)])
    else:
      # Run eval, validation & selfplay sequentially.
      model_win_rate = wait(evaluate_trained_model(state))
      wait(validate(state, holdout_glob))
      wait(selfplay(state))

    if model_win_rate >= FLAGS.gating_win_rate:
      # Promote the trained model to the best model and increment the generation
      # number.
      state.best_model_name = state.train_model_name
      state.gen_num += 1
895183
def compute(self, x):
        '''
        
        '''
        q_learning = copy(self.__greedy_q_learning)
        q_learning.epsilon_greedy_rate = x[0]
        q_learning.alpha_value = x[1]
        q_learning.gamma_value = x[2]
        if self.__init_state_key is not None:
            q_learning.learn(state_key=self.__init_state_key, limit=int(x[3]))
        else:
            q_learning.learn(limit=x[3])
        q_sum = q_learning.q_df.q_value.sum()
        if q_sum != 0:
            cost = q_learning.q_df.shape[0] / q_sum
        else:
            cost = q_learning.q_df.shape[0] / 1e-4

        return cost
279417
def _initilize(self, state: State) -> State:
        """"""

        targets = {}
        for pc, instr in enumerate(self):
            if isinstance(instr, Label):
                targets[instr.target] = pc

        state = state.update({PC: 0,
                              TARGETS: targets,
                              NAMEDGATES: STDGATES.copy()})
        return state
1355659
public void updateKey(N node){
        node.getKey().update(node.getG(), node.getV(), heuristicFunction.estimate(node.state()), epsilon, add, scale);
    }
1345513
def init_instance(self, key):
        
        with self._mor_lock:
            if key not in self._mor:
                self._mor[key] = {}
161818
public void learn(RNNSearch<double[], double[]> range, double[][] samples, double radius, int Nz, double sigma) {
        TrainSample[] training = new TrainSample[samples.length];
        for (int i = 0; i < samples.length; i++) {
            training[i] = new TrainSample();
            training[i].query = samples[i];
            training[i].neighbors = new ArrayList<>();
            ArrayList<Neighbor<double[], double[]>> neighbors = new ArrayList<>();
            range.range(training[i].query, radius, neighbors);
            for (Neighbor<double[], double[]> n : neighbors) {
                training[i].neighbors.add(keys.get(n.index));
            }
        }

        model = new ArrayList<>(hash.size());
        for (int i = 0; i < hash.size(); i++) {
            model.add(new PosterioriModel(hash.get(i), training, Nz, sigma));
        }
    }
547328
def inference(self, limit=1000):
        '''
        
        '''
        route_list = []
        memory_list = []
        state_key = self.__start_point_tuple
        x, y = state_key
        end_x, end_y = self.__end_point_tuple
        for i in range(limit):
            q_df = self.q_df[self.q_df.state_key == state_key]
            if len(memory_list):
                q_df = q_df[~q_df.action_key.isin(memory_list)]
            if q_df.shape[0] > 1:
                q_df = q_df.sort_values(by=["q_value"], ascending=False)
                action_key = q_df.iloc[0, :]["action_key"]
                q_value = q_df.iloc[0, :]["q_value"]
            elif q_df.shape[0] == 1:
                action_key = q_df.action_key.values[0]
                q_value = q_df.q_value.values[0]
            else:
                action_key_list = self.extract_possible_actions(state_key)
                action_key_list = [v for v in action_key_list if v not in memory_list]
                q_value = 0.0
                if len(action_key_list):
                    action_key = random.choice(action_key_list)
                    _q_df = q_df[q_df.action_key == action_key]
                    if _q_df.shape[0]:
                        q_value = _q_df.q_value.values[0]

            state_key = self.update_state(
                state_key=state_key,
                action_key=action_key
            )
            x, y = state_key
            route_list.append((x, y, q_value))
            memory_list.append(state_key)
            if self.check_the_end_flag(state_key) is True:
                break

        return route_list
1049173
public void learn(int iterations) {
        double[][] Y = coordinates;
        int n = Y.length;
        int d = Y[0].length;

        int nprocs = MulticoreExecutor.getThreadPoolSize();
        int chunk = n / nprocs;
        List<SNETask> tasks = new ArrayList<>();
        for (int i = 0; i < nprocs; i++) {
            int start = i * chunk;
            int end = i == nprocs-1 ? n : (i+1) * chunk;
            SNETask task = new SNETask(start, end);
            tasks.add(task);
        }

        for (int iter = 1; iter <= iterations; iter++, totalIter++) {
            Math.pdist(Y, Q, true, false);
            Qsum = 0.0;
            for (int i = 0; i < n; i++) {
                double[] Qi = Q[i];
                for (int j = 0; j < i; j++) {
                    double q = 1.0 / (1.0 + Qi[j]);
                    Qi[j] = q;
                    Q[j][i] = q;
                    Qsum += q;
                }
            }
            Qsum *= 2.0;

            try {
                MulticoreExecutor.run(tasks);
            } catch (Exception e) {
                logger.error("t-SNE iteration task fails: {}", e);
            }

            if (totalIter == momentumSwitchIter) {
                momentum = finalMomentum;
                for (int i = 0; i < n; i++) {
                    double[] Pi = P[i];
                    for (int j = 0; j < n; j++) {
                        Pi[j] /= 12.0;
                    }
                }
            }

            // Compute current value of cost function
            if (iter % 50 == 0)   {
                double C = 0.0;
                for (int i = 0; i < n; i++) {
                    double[] Pi = P[i];
                    double[] Qi = Q[i];
                    for (int j = 0; j < i; j++) {
                        double p = Pi[j];
                        double q = Qi[j] / Qsum;
                        if (Double.isNaN(q) || q < 1E-16) q = 1E-16;
                        C += p * Math.log2(p / q);
                    }
                }
                logger.info("Error after {} iterations: {}", totalIter, 2 * C);
            }
        }

        // Make solution zero-mean
        double[] colMeans = Math.colMeans(Y);
        for (int i = 0; i < n; i++) {
            double[] Yi = Y[i];
            for (int j = 0; j < d; j++) {
                Yi[j] -= colMeans[j];
            }
        }
    }
1241461
def start(self, key):
        
        start_d = self._get_limiter(key).start()
        self._cleanup_limiter(key)
        return start_d
1118380
public void rnnSetPreviousState(int layer, Map<String, INDArray> state) {
        rnnSetPreviousState(layers[layer].conf().getLayer().getLayerName(), state);
    }
371265
def update_policy(self):
        
        self.demonstration_buffer.update_buffer.shuffle()
        batch_losses = []
        num_batches = min(len(self.demonstration_buffer.update_buffer['actions']) //
                          self.n_sequences, self.batches_per_epoch)
        for i in range(num_batches):
            update_buffer = self.demonstration_buffer.update_buffer
            start = i * self.n_sequences
            end = (i + 1) * self.n_sequences
            mini_batch = update_buffer.make_mini_batch(start, end)
            run_out = self.policy.update(mini_batch, self.n_sequences)
            loss = run_out['policy_loss']
            batch_losses.append(loss)
        if len(batch_losses) > 0:
            self.stats['Losses/Cloning Loss'].append(np.mean(batch_losses))
        else:
            self.stats['Losses/Cloning Loss'].append(0)