Sets the learning rate to the initial LR decayed by 10 every 30 epochs
359120
def adjust_learning_rate(optimizer, epoch, gammas, schedule):
  """"""
  lr = args.learning_rate
  assert len(gammas) == len(schedule), "length of gammas and schedule should be equal"
  for (gamma, step) in zip(gammas, schedule):
    if (epoch >= step):
      lr = lr * gamma
    else:
      break
  for param_group in optimizer.param_groups:
    param_group['lr'] = lr
  return lr
655331
def _build_train_op(self):
        """"""
        num_gpus = self.hps.num_gpus if self.hps.num_gpus != 0 else 1
        # The learning rate schedule is dependent on the number of gpus.
        boundaries = [int(20000 * i / np.sqrt(num_gpus)) for i in range(2, 5)]
        values = [0.1, 0.01, 0.001, 0.0001]
        self.lrn_rate = tf.train.piecewise_constant(self.global_step,
                                                    boundaries, values)
        tf.summary.scalar("learning rate", self.lrn_rate)

        if self.hps.optimizer == "sgd":
            optimizer = tf.train.GradientDescentOptimizer(self.lrn_rate)
        elif self.hps.optimizer == "mom":
            optimizer = tf.train.MomentumOptimizer(self.lrn_rate, 0.9)

        apply_op = optimizer.minimize(self.cost, global_step=self.global_step)
        train_ops = [apply_op] + self._extra_train_ops
        self.train_op = tf.group(*train_ops)
        self.variables = ray.experimental.tf_utils.TensorFlowVariables(
            self.train_op)
221528
def train(net, X_train, y_train, epochs, verbose_epoch, learning_rate,
          weight_decay, batch_size):
    """"""
    dataset_train = gluon.data.ArrayDataset(X_train, y_train)
    data_iter_train = gluon.data.DataLoader(dataset_train, batch_size,
                                            shuffle=True)
    trainer = gluon.Trainer(net.collect_params(), 'adam',
                            {'learning_rate': learning_rate,
                             'wd': weight_decay})
    net.initialize(force_reinit=True)
    for epoch in range(epochs):
        for data, label in data_iter_train:
            with autograd.record():
                output = net(data)
                loss = square_loss(output, label)
            loss.backward()
            trainer.step(batch_size)
            avg_loss = get_rmse_log(net, X_train, y_train)
        if epoch > verbose_epoch:
            print("Epoch %d, train loss: %f" % (epoch, avg_loss))
    return avg_loss
228203
def log_learning_rates(self,
                           model: Model,
                           optimizer: torch.optim.Optimizer):
        
        if self._should_log_learning_rate:
            # optimizer stores lr info keyed by parameter tensor
            # we want to log with parameter name
            names = {param: name for name, param in model.named_parameters()}
            for group in optimizer.param_groups:
                if 'lr' not in group:
                    continue
                rate = group['lr']
                for param in group['params']:
                    # check whether params has requires grad or not
                    effective_rate = rate * float(param.requires_grad)
                    self.add_train_scalar("learning_rate/" + names[param], effective_rate)
295540
def on_train_begin(self, epoch:int, **kwargs:Any)->None:
        ""
        res = {'epoch':self.start_epoch} if self.start_epoch is not None else None
        self.start_epoch = ifnone(self.start_epoch, epoch)
        self.scheds = [p.scheds for p in self.phases]
        self.opt = self.learn.opt
        for k,v in self.scheds[0].items(): 
            v.restart()
            self.opt.set_stat(k, v.start)
        self.idx_s = 0
        return res
759384
def get_learning_rate(self, iter):
        '''
        
        '''

        return self.init_lr * (self.gamma ** (iter // self.iter_interval))
1325949
def train(data_train, model, nsp_loss, mlm_loss, vocab_size, ctx):
    """"""
    hvd.broadcast_parameters(model.collect_params(), root_rank=0)

    mlm_metric = nlp.metric.MaskedAccuracy()
    nsp_metric = nlp.metric.MaskedAccuracy()
    mlm_metric.reset()
    nsp_metric.reset()

    logging.debug('Creating distributed trainer...')
    lr = args.lr
    optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
    if args.dtype == 'float16':
        optim_params['multi_precision'] = True

    dynamic_loss_scale = args.dtype == 'float16'
    if dynamic_loss_scale:
        loss_scale_param = {'scale_window': 2000 / num_workers}
    else:
        loss_scale_param = None
    trainer = hvd.DistributedTrainer(model.collect_params(), 'bertadam', optim_params)
    fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale,
                               loss_scaler_params=loss_scale_param)

    if args.ckpt_dir and args.start_step:
        trainer.load_states(os.path.join(args.ckpt_dir, '%07d.states'%args.start_step))

    accumulate = args.accumulate
    num_train_steps = args.num_steps
    warmup_ratio = args.warmup_ratio
    num_warmup_steps = int(num_train_steps * warmup_ratio)
    params = [p for p in model.collect_params().values() if p.grad_req != 'null']
    param_dict = model.collect_params()

    # Do not apply weight decay on LayerNorm and bias terms
    for _, v in model.collect_params('.*beta|.*gamma|.*bias').items():
        v.wd_mult = 0.0
    if accumulate > 1:
        for p in params:
            p.grad_req = 'add'

    train_begin_time = time.time()
    begin_time = time.time()
    running_mlm_loss, running_nsp_loss = 0, 0
    running_num_tks = 0
    batch_num = 0
    step_num = args.start_step

    logging.debug('Training started')
    while step_num < num_train_steps:
        for _, dataloader in enumerate(data_train):
            if step_num >= num_train_steps:
                break

            # create dummy data loader if needed
            if args.dummy_data_len:
                target_shape = (args.batch_size, args.dummy_data_len)
                dataloader = get_dummy_dataloader(dataloader, target_shape)

            for _, data_batch in enumerate(dataloader):
                if step_num >= num_train_steps:
                    break
                if batch_num % accumulate == 0:
                    step_num += 1
                    # if accumulate > 1, grad_req is set to 'add', and zero_grad is required
                    if accumulate > 1:
                        param_dict.zero_grad()
                    # update learning rate
                    if step_num <= num_warmup_steps:
                        new_lr = lr * step_num / num_warmup_steps
                    else:
                        offset = lr * step_num / num_train_steps
                        new_lr = lr - offset
                    trainer.set_learning_rate(new_lr)
                    if args.profile:
                        profile(step_num, 10, 14, profile_name=args.profile + str(rank))

                # load data
                if args.use_avg_len:
                    data_list = [[seq.as_in_context(context) for seq in shard]
                                 for context, shard in zip([ctx], data_batch)]
                else:
                    data_list = list(split_and_load(data_batch, [ctx]))
                data = data_list[0]

                # forward
                with mx.autograd.record():
                    (ls, ns_label, classified, masked_id, decoded, \
                     masked_weight, ls1, ls2, valid_len) = forward(data, model, mlm_loss,
                                                                   nsp_loss, vocab_size, args.dtype)
                    ls = ls / accumulate
                    # backward
                    if args.dtype == 'float16':
                        fp16_trainer.backward(ls)
                    else:
                        ls.backward()

                running_mlm_loss += ls1.as_in_context(mx.cpu())
                running_nsp_loss += ls2.as_in_context(mx.cpu())
                running_num_tks += valid_len.sum().as_in_context(mx.cpu())

                # update
                if (batch_num + 1) % accumulate == 0:
                    # step() performs 3 things:
                    # 1. allreduce gradients from all workers
                    # 2. checking the global_norm of gradients and clip them if necessary
                    # 3. averaging the gradients and apply updates
                    fp16_trainer.step(1, max_norm=1*num_workers)

                nsp_metric.update([ns_label], [classified])
                mlm_metric.update([masked_id], [decoded], [masked_weight])

                # logging
                if (step_num + 1) % (args.log_interval) == 0 and (batch_num + 1) % accumulate == 0:
                    log(begin_time, running_num_tks, running_mlm_loss / accumulate,
                        running_nsp_loss / accumulate, step_num, mlm_metric, nsp_metric,
                        trainer, args.log_interval)
                    begin_time = time.time()
                    running_mlm_loss = running_nsp_loss = running_num_tks = 0
                    mlm_metric.reset_local()
                    nsp_metric.reset_local()

                # saving checkpoints
                if args.ckpt_dir and (step_num + 1) % (args.ckpt_interval) == 0 \
                   and (batch_num + 1) % accumulate == 0 and local_rank == 0:
                    save_params(step_num, model, trainer, args.ckpt_dir)

                batch_num += 1

    if local_rank == 0:
        save_params(step_num, model, trainer, args.ckpt_dir)
    mx.nd.waitall()
    train_end_time = time.time()
    logging.info('Train cost={:.1f}s'.format(train_end_time - train_begin_time))
589301
def on_train_begin(self, n_epochs:int, epoch:int, **kwargs:Any)->None:
        ""
        res = {'epoch':self.start_epoch} if self.start_epoch is not None else None
        self.start_epoch = ifnone(self.start_epoch, epoch)
        self.tot_epochs = ifnone(self.tot_epochs, n_epochs)
        n = len(self.learn.data.train_dl) * self.tot_epochs
        a1 = int(n * self.pct_start)
        a2 = n-a1
        self.phases = ((a1, annealing_cos), (a2, annealing_cos))
        low_lr = self.lr_max/self.div_factor
        self.lr_scheds = self.steps((low_lr, self.lr_max), (self.lr_max, self.lr_max/self.final_div))
        self.mom_scheds = self.steps(self.moms, (self.moms[1], self.moms[0]))
        self.opt = self.learn.opt
        self.opt.lr,self.opt.mom = self.lr_scheds[0].start,self.mom_scheds[0].start
        self.idx_s = 0
        return res
490406
def train(self):
        
        self.stamp_start = time.time()
        for iteration, batch in tqdm.tqdm(enumerate(self.iter_train),
                                          desc='train', total=self.max_iter,
                                          ncols=80):
            self.epoch = self.iter_train.epoch
            self.iteration = iteration

            ############
            # validate #
            ############

            if self.interval_validate and \
                    self.iteration % self.interval_validate == 0:
                self.validate()

            #########
            # train #
            #########

            batch = map(datasets.transform_lsvrc2012_vgg16, batch)
            in_vars = utils.batch_to_vars(batch, device=self.device)
            self.model.zerograds()
            loss = self.model(*in_vars)

            if loss is not None:
                loss.backward()
                self.optimizer.update()

                lbl_true = zip(*batch)[1]
                lbl_pred = chainer.functions.argmax(self.model.score, axis=1)
                lbl_pred = chainer.cuda.to_cpu(lbl_pred.data)
                acc = utils.label_accuracy_score(
                    lbl_true, lbl_pred, self.model.n_class)
                self._write_log(**{
                    'epoch': self.epoch,
                    'iteration': self.iteration,
                    'elapsed_time': time.time() - self.stamp_start,
                    'train/loss': float(loss.data),
                    'train/acc': acc[0],
                    'train/acc_cls': acc[1],
                    'train/mean_iu': acc[2],
                    'train/fwavacc': acc[3],
                })

            if iteration >= self.max_iter:
                self._save_model()
                break
1228671
def initialize(self, params, repetition):
    
    self.name = params["name"]
    self.dataDir = params.get("datadir", "data")
    self.seed = params.get("seed", 42) + repetition

    torch.manual_seed(self.seed)
    np.random.seed(self.seed)
    random.seed(self.seed)

    # Training
    self.epochs = params.get("epochs", 1)
    self.batch_size = params.get("batch_size", 128)
    self.batches_in_epoch = params.get("batches_in_epoch", 100000)
    self.first_epoch_batch_size = params.get("first_epoch_batch_size",
                                             self.batch_size)
    self.batches_in_first_epoch = params.get("batches_in_first_epoch",
                                             self.batches_in_epoch)

    # Testing
    self.test_batch_size = params.get("test_batch_size", 1000)
    self.test_batches_in_epoch = params.get("test_batches_in_epoch", 100000)
    self.noise_values = map(float,
                            params.get("noise_values", "0.0, 0.1").split(", "))
    self.best_noise_score = 0.0
    self.best_epoch = -1

    # Optimizer
    self.optimizer_class = eval(params.get("optimizer", "torch.optim.SGD"))
    self.lr = params.get("learning_rate", 0.05)
    self.momentum = params.get("momentum", 0.9)
    self.weight_decay = params.get("weight_decay", 0.0)
    self.optimizer_params = eval(params.get("optimizer_params", "{}"))
    self.lr_scheduler_gamma = params.get("lr_scheduler_gamma", 0.9)
    self.loss_function = eval(params.get("loss_function",
                                         "torch.nn.functional.cross_entropy"))

    # Network parameters
    self.conv1_sparsity = params.get("conv1_sparsity", 1.0)
    self.network_type = params.get("network_type", "sparse")
    self.growth_rate = params.get("growth_rate", 12)
    self.nblocks = map(int,
                       params.get("nblocks", "6, 12, 24, 16").split(", "))
    self.k_inference_factor = params.get("k_inference_factor", 1.5)

    self.dense_sparsities = map(float,
                                params.get("dense_sparsities",
                                           "1.0, 1.0, 1.0, 1.0").split(", "))
    self.transition_sparsities = map(float,
                                     params.get("transition_sparsities",
                                                "0.1, 0.1, 0.2").split(", "))
    self.linear_sparsity = params.get("linear_sparsity", 0.0)
    self.linear_weight_sparsity = params.get("linear_weight_sparsity", 0.3)
    self.linear_n = params.get("linear_n", 500)
    self.avg_pool_size = params.get("avg_pool_size", 2)
    self.dense_c1_out_planes = params.get("dense_c1_out_planes", 4*self.growth_rate)
832712
def train(epochs, ctx):
    """"""
    if isinstance(ctx, mx.Context):
        ctx = [ctx]
    net.initialize(mx.init.Xavier(magnitude=2), ctx=ctx)

    opt_options = {'learning_rate': opt.lr, 'wd': opt.wd}
    if opt.optimizer == 'sgd':
        opt_options['momentum'] = 0.9
    if opt.optimizer == 'adam':
        opt_options['epsilon'] = 1e-7
    trainer = gluon.Trainer(net.collect_params(), opt.optimizer,
                            opt_options,
                            kvstore=opt.kvstore)
    if opt.lr_beta > 0.0:
        # Jointly train class-specific beta.
        # See "sampling matters in deep embedding learning" paper for details.
        beta.initialize(mx.init.Constant(opt.beta), ctx=ctx)
        trainer_beta = gluon.Trainer([beta], 'sgd',
                                     {'learning_rate': opt.lr_beta, 'momentum': 0.9},
                                     kvstore=opt.kvstore)

    loss = MarginLoss(margin=opt.margin, nu=opt.nu)

    best_val = 0.0
    for epoch in range(epochs):
        tic = time.time()
        prev_loss, cumulative_loss = 0.0, 0.0

        # Learning rate schedule.
        trainer.set_learning_rate(get_lr(opt.lr, epoch, steps, opt.factor))
        logging.info('Epoch %d learning rate=%f', epoch, trainer.learning_rate)
        if opt.lr_beta > 0.0:
            trainer_beta.set_learning_rate(get_lr(opt.lr_beta, epoch, steps, opt.factor))
            logging.info('Epoch %d beta learning rate=%f', epoch, trainer_beta.learning_rate)

        # Inner training loop.
        for i in range(200):
            batch = train_data.next()
            data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0)
            label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0)

            Ls = []
            with ag.record():
                for x, y in zip(data, label):
                    a_indices, anchors, positives, negatives, _ = net(x)

                    if opt.lr_beta > 0.0:
                        L = loss(anchors, positives, negatives, beta, y[a_indices])
                    else:
                        L = loss(anchors, positives, negatives, opt.beta, None)

                    # Store the loss and do backward after we have done forward
                    # on all GPUs for better speed on multiple GPUs.
                    Ls.append(L)
                    cumulative_loss += nd.mean(L).asscalar()

                for L in Ls:
                    L.backward()

            # Update.
            trainer.step(batch.data[0].shape[0])
            if opt.lr_beta > 0.0:
                trainer_beta.step(batch.data[0].shape[0])

            if (i+1) % opt.log_interval == 0:
                logging.info('[Epoch %d, Iter %d] training loss=%f' % (
                    epoch, i+1, cumulative_loss - prev_loss))
                prev_loss = cumulative_loss

        logging.info('[Epoch %d] training loss=%f'%(epoch, cumulative_loss))
        logging.info('[Epoch %d] time cost: %f'%(epoch, time.time()-tic))

        names, val_accs = test(ctx)
        for name, val_acc in zip(names, val_accs):
            logging.info('[Epoch %d] validation: %s=%f'%(epoch, name, val_acc))

        if val_accs[0] > best_val:
            best_val = val_accs[0]
            logging.info('Saving %s.' % opt.save_model_prefix)
            net.save_parameters('%s.params' % opt.save_model_prefix)
    return best_val
602932
def _prepare(self, **kwargs):
        '''
        '''
        self.learning_rate = util.as_float(kwargs.pop('learning_rate', 1e-4))
        self.momentum = kwargs.pop('momentum', 0)
        self.nesterov = kwargs.pop('nesterov', False)
        self.patience = kwargs.get('patience', 5)
        self.validate_every = kwargs.pop('validate_every', 10)
        self.min_improvement = kwargs.pop('min_improvement', 0)
        self.max_gradient_norm = kwargs.pop('max_gradient_norm', 0)
        self.max_gradient_elem = kwargs.pop('max_gradient_elem', 0)

        util.log_param('patience', self.patience)
        util.log_param('validate_every', self.validate_every)
        util.log_param('min_improvement', self.min_improvement)
        util.log_param('max_gradient_norm', self.max_gradient_norm)
        util.log_param('max_gradient_elem', self.max_gradient_elem)
        util.log_param('learning_rate', self.learning_rate)
        util.log_param('momentum', self.momentum)
        util.log_param('nesterov', self.nesterov)
780318
def _update_pref(self, lmin, lmax, cur):
        
        rate_of_change_positive = 10
        rate_of_change_negative = 2
        add_positive = 10
        add_negative = 2
        if lmin == lmax:
            self.rating -= int(abs(lmin - cur)*100) / 10
        else:
            if lmin <= cur:
                self.rating += (int(abs(lmin - cur)*rate_of_change_positive)) + add_positive
            else:
                self.rating -= (int(abs(lmin - cur)*rate_of_change_negative)) + add_negative
            if lmax >= cur:
                self.rating += (int(abs(lmax - cur)*rate_of_change_positive)) + add_positive
            else:
                self.rating -= (int(abs(lmax - cur)*rate_of_change_negative)) + add_negative
672833
def do_training(num_epoch, optimizer, kvstore, learning_rate, model_prefix, decay):
    """"""
    summary_writer = SummaryWriter(args.tblog_dir)
    lr_scheduler = SimpleLRScheduler(learning_rate)
    optimizer_params = {'lr_scheduler': lr_scheduler}
    module.init_params()
    module.init_optimizer(kvstore=kvstore,
                          optimizer=optimizer,
                          optimizer_params=optimizer_params)
    n_epoch = 0
    while True:
        if n_epoch >= num_epoch:
            break
        train_iter.reset()
        val_iter.reset()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(train_iter):
            module.forward_backward(data_batch)
            module.update()
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        train_acc, train_loss, train_recon_err = loss_metric.get_name_value()
        loss_metric.reset()
        for n_batch, data_batch in enumerate(val_iter):
            module.forward(data_batch)
            module.update_metric(loss_metric, data_batch.label)
            loss_metric.get_batch_log(n_batch)
        val_acc, val_loss, val_recon_err = loss_metric.get_name_value()

        summary_writer.add_scalar('train_acc', train_acc, n_epoch)
        summary_writer.add_scalar('train_loss', train_loss, n_epoch)
        summary_writer.add_scalar('train_recon_err', train_recon_err, n_epoch)
        summary_writer.add_scalar('val_acc', val_acc, n_epoch)
        summary_writer.add_scalar('val_loss', val_loss, n_epoch)
        summary_writer.add_scalar('val_recon_err', val_recon_err, n_epoch)

        print('Epoch[%d] train acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, train_acc, train_loss,
                                                                        train_recon_err))
        print('Epoch[%d] val acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, val_acc, val_loss, val_recon_err))
        print('SAVE CHECKPOINT')

        module.save_checkpoint(prefix=model_prefix, epoch=n_epoch)
        n_epoch += 1
        lr_scheduler.learning_rate = learning_rate * (decay ** n_epoch)
476147
def optimize(self, loss, num_async_replicas=1, use_tpu=False):
    """"""
    lr = learning_rate.learning_rate_schedule(self.hparams)
    if num_async_replicas > 1:
      log_info("Dividing learning rate by num_async_replicas: %d",
               num_async_replicas)
    lr /= math.sqrt(float(num_async_replicas))
    train_op = optimize.optimize(loss, lr, self.hparams, use_tpu=use_tpu)
    return train_op
1064340
def create(model_config, reinforcer, optimizer, storage, total_frames, batches_per_epoch,
           callbacks=None, scheduler=None, openai_logging=False):
    
    from vel.openai.baselines import logger
    logger.configure(dir=model_config.openai_dir())

    return RlTrainCommand(
        model_config=model_config,
        reinforcer=reinforcer,
        optimizer_factory=optimizer,
        scheduler_factory=scheduler,
        storage=storage,
        callbacks=callbacks,
        total_frames=int(float(total_frames)),
        batches_per_epoch=int(batches_per_epoch),
        openai_logging=openai_logging
    )
160428
def _learning_rate_warmup(warmup_steps, warmup_schedule="exp", hparams=None):
  """"""
  if not warmup_steps:
    return tf.constant(1.)

  tf.logging.info("Applying %s learning rate warmup for %d steps",
                  warmup_schedule, warmup_steps)

  warmup_steps = tf.to_float(warmup_steps)
  global_step = _global_step(hparams)

  if warmup_schedule == "exp":
    return tf.exp(tf.log(0.01) / warmup_steps)**(warmup_steps - global_step)
  else:
    assert warmup_schedule == "linear"
    start = tf.constant(0.35)
    return ((tf.constant(1.) - start) / warmup_steps) * global_step + start