Get learning rate based on schedule.
161225
def get_lr(lr, epoch, steps, factor):
    """"""
    for s in steps:
        if epoch >= s:
            lr *= factor
    return lr
866047
def scheduling_blocks():
    """"""
    sbi_list = SchedulingBlockInstanceList()
    return dict(active=sbi_list.active,
                completed=sbi_list.completed,
                aborted=sbi_list.aborted)
442797
def logger(self, iteration, ret):
    """"""
    print("Learning rate: {:f}".format(self.lr_scheduler.get_lr()[0]))
    entropies = getEntropies(self.model)
    print("Entropy and max entropy: ", float(entropies[0]), entropies[1])
    print("Training time for epoch=", self.epoch_train_time)
    for noise in self.noise_values:
      print("Noise= {:3.2f}, loss = {:5.4f}, Accuracy = {:5.3f}%".format(
        noise, ret[noise]["loss"], 100.0*ret[noise]["accuracy"]))
    print("Full epoch time =", self.epoch_time)
    if ret[0.0]["accuracy"] > 0.7:
      self.best_noise_score = max(ret[0.1]["accuracy"], self.best_noise_score)
      self.best_epoch = iteration
880561
def value(self, t):
        """"""
        fraction = min(float(t) / max(1, self.schedule_timesteps), 1.0)
        return self.initial_p + fraction * (self.final_p - self.initial_p)
617828
def parse_schedule_str(schedule_str: str) -> List[Tuple[float, int]]:
        
        schedule = list()
        for step in schedule_str.split(","):
            rate, num_updates = step.split(":")
            schedule.append((float(rate), int(num_updates)))
        return schedule
672833
def do_training(num_epoch, optimizer, kvstore, learning_rate, model_prefix, decay):
    """"""
    summary_writer = SummaryWriter(args.tblog_dir)
    lr_scheduler = SimpleLRScheduler(learning_rate)
    optimizer_params = {'lr_scheduler': lr_scheduler}
    module.init_params()
    module.init_optimizer(kvstore=kvstore,
                          optimizer=optimizer,
                          optimizer_params=optimizer_params)
    n_epoch = 0
    while True:
        if n_epoch >= num_epoch:
            break
        train_iter.reset()
        val_iter.reset()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(train_iter):
            module.forward_backward(data_batch)
            module.update()
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        train_acc, train_loss, train_recon_err = loss_metric.get_name_value()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(val_iter):
            module.forward(data_batch)
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        val_acc, val_loss, val_recon_err = loss_metric.get_name_value()

        summary_writer.add_scalar('train_acc', train_acc, n_epoch)
        summary_writer.add_scalar('train_loss', train_loss, n_epoch)
        summary_writer.add_scalar('train_recon_err', train_recon_err, n_epoch)
        summary_writer.add_scalar('val_acc', val_acc, n_epoch)
        summary_writer.add_scalar('val_loss', val_loss, n_epoch)
        summary_writer.add_scalar('val_recon_err', val_recon_err, n_epoch)

        print('Epoch[%d] train acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, train_acc, train_loss,
                                                                        train_recon_err))
        print('Epoch[%d] val acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, val_acc, val_loss, val_recon_err))
        print('SAVE CHECKPOINT')

        module.save_checkpoint(prefix=model_prefix, epoch=n_epoch)
        n_epoch += 1
        lr_scheduler.learning_rate = learning_rate * (decay ** n_epoch)
1066522
def createLearningRateScheduler(self, optimizer):
    
    if self.lr_scheduler_class is None:
      return None

    return self.lr_scheduler_class(optimizer, **self.lr_scheduler_params)
5629
def train():
    """"""
    trainer = gluon.Trainer(model.collect_params(), args.optimizer, {'learning_rate': args.lr})

    train_data_loader, val_data_loader, test_data_loader \
        = dataprocessor.make_dataloader(data_train, data_val, data_test, args)

    best_valid_bleu = 0.0
    for epoch_id in range(args.epochs):
        log_avg_loss = 0
        log_avg_gnorm = 0
        log_wc = 0
        log_start_time = time.time()
        for batch_id, (src_seq, tgt_seq, src_valid_length, tgt_valid_length)\
                in enumerate(train_data_loader):
            # logging.info(src_seq.context) Context suddenly becomes GPU.
            src_seq = src_seq.as_in_context(ctx)
            tgt_seq = tgt_seq.as_in_context(ctx)
            src_valid_length = src_valid_length.as_in_context(ctx)
            tgt_valid_length = tgt_valid_length.as_in_context(ctx)
            with mx.autograd.record():
                out, _ = model(src_seq, tgt_seq[:, :-1], src_valid_length, tgt_valid_length - 1)
                loss = loss_function(out, tgt_seq[:, 1:], tgt_valid_length - 1).mean()
                loss = loss * (tgt_seq.shape[1] - 1) / (tgt_valid_length - 1).mean()
                loss.backward()
            grads = [p.grad(ctx) for p in model.collect_params().values()]
            gnorm = gluon.utils.clip_global_norm(grads, args.clip)
            trainer.step(1)
            src_wc = src_valid_length.sum().asscalar()
            tgt_wc = (tgt_valid_length - 1).sum().asscalar()
            step_loss = loss.asscalar()
            log_avg_loss += step_loss
            log_avg_gnorm += gnorm
            log_wc += src_wc + tgt_wc
            if (batch_id + 1) % args.log_interval == 0:
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ppl={:.4f}, gnorm={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'
                             .format(epoch_id, batch_id + 1, len(train_data_loader),
                                     log_avg_loss / args.log_interval,
                                     np.exp(log_avg_loss / args.log_interval),
                                     log_avg_gnorm / args.log_interval,
                                     wps / 1000, log_wc / 1000))
                log_start_time = time.time()
                log_avg_loss = 0
                log_avg_gnorm = 0
                log_wc = 0
        valid_loss, valid_translation_out = evaluate(val_data_loader)
        valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
        logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                     .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
        test_loss, test_translation_out = evaluate(test_data_loader)
        test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
        dataprocessor.write_sentences(valid_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_valid_out.txt').format(epoch_id))
        dataprocessor.write_sentences(test_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_test_out.txt').format(epoch_id))
        if valid_bleu_score > best_valid_bleu:
            best_valid_bleu = valid_bleu_score
            save_path = os.path.join(args.save_dir, 'valid_best.params')
            logging.info('Save best parameters to {}'.format(save_path))
            model.save_parameters(save_path)
        if epoch_id + 1 >= (args.epochs * 2) // 3:
            new_lr = trainer.learning_rate * args.lr_update_factor
            logging.info('Learning rate change to {}'.format(new_lr))
            trainer.set_learning_rate(new_lr)
    if os.path.exists(os.path.join(args.save_dir, 'valid_best.params')):
        model.load_parameters(os.path.join(args.save_dir, 'valid_best.params'))
    valid_loss, valid_translation_out = evaluate(val_data_loader)
    valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
    logging.info('Best model valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                 .format(valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
    test_loss, test_translation_out = evaluate(test_data_loader)
    test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
    logging.info('Best model test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                 .format(test_loss, np.exp(test_loss), test_bleu_score * 100))
    dataprocessor.write_sentences(valid_translation_out,
                                  os.path.join(args.save_dir, 'best_valid_out.txt'))
    dataprocessor.write_sentences(test_translation_out,
                                  os.path.join(args.save_dir, 'best_test_out.txt'))
438021
def learn(self, x):
        """"""
        y = self.encode(x)
        self.update_statistics([y]) 
        self.update_weights([x],[y])
        return y
840949
def revnet_range(rhp):
  """"""
  rhp.set_float('learning_rate', 0.05, 0.2, scale=rhp.LOG_SCALE)
  rhp.set_float('weight_decay', 1e-5, 1e-3, scale=rhp.LOG_SCALE)
  rhp.set_discrete('num_channels_init_block', [64, 128])
  return rhp
725730
def train(self, X_train, Y_train, X_test, Y_test):
        

        while True:
            print(1)
            time.sleep(1)
            if random.randint(0, 9) >= 5:
                break
278798
def learningCurve(expPath, suite):
  
  print("\nLEARNING CURVE ================",expPath,"=====================")
  try:
    headers=["testerror","totalCorrect","elapsedTime","entropy"]
    result = suite.get_value(expPath, 0, headers, "all")
    info = []
    for i,v in enumerate(zip(result["testerror"],result["totalCorrect"],
                             result["elapsedTime"],result["entropy"])):
      info.append([i, v[0], v[1], int(v[2]), v[3]])
    headers.insert(0,"iteration")
    print(tabulate(info, headers=headers, tablefmt="grid"))
  except:
    print("Couldn't load experiment",expPath)
356602
def legacy_learning_rate_schedule(hparams):
  """"""
  step_num = _global_step(hparams)
  warmup_steps = tf.to_float(hparams.learning_rate_warmup_steps)
  if hparams.learning_rate_decay_scheme == "noam":
    ret = 5000.0 * hparams.hidden_size**-0.5 * tf.minimum(
        (step_num + 1) * warmup_steps**-1.5, (step_num + 1)**-0.5)
  else:
    warmup_steps = hparams.learning_rate_warmup_steps
    warmup = _learning_rate_warmup(warmup_steps, hparams=hparams)
    decay = _learning_rate_decay(hparams, warmup_steps)
    ret = tf.where(step_num < warmup_steps, warmup, decay)
  optimizer_correction = 0.002 if "adam" in hparams.optimizer else 1.0
  tf.logging.info("Base learning rate: %f", hparams.learning_rate)
  return ret * optimizer_correction * hparams.learning_rate
20387
def _fromiter(it, dtype, count, progress, log):
    """"""
    if progress > 0:
        it = _iter_withprogress(it, progress, log)
    if count is not None:
        a = np.fromiter(it, dtype=dtype, count=count)
    else:
        a = np.fromiter(it, dtype=dtype)
    return a
1047387
@Override
    public double getLearningRate(int iterCount, int i) {
        // We use the learning rate suggested in Leon Bottou's (2012) SGD Tricks paper.
        // 
        // \gamma_t = \frac{\gamma_0}{(1 + \gamma_0 \lambda t)^p}
        //
        // For SGD p = 1.0, for ASGD p = 0.75
        if (prm.power == 1.0) {
            return prm.initialLr / (1 + prm.initialLr * prm.lambda * iterCount);
        } else {
            return prm.initialLr / Math.pow(1 + prm.initialLr * prm.lambda * iterCount, prm.power);
        }
    }
1242538
def get(self, sched_rule_id):
        """"""
        path = '/'.join(['schedulerule', sched_rule_id])
        return self.rachio.get(path)
334686
def get_learning_rate(self, iter):
        '''
        
        '''
        return self.init_lr * ((math.cos(iter * 1.0 / (self.max_iter) * math.pi) + 1.0) * 0.5)