Train the model using Caffe operator in MXNet
504127
def fit(args, network, data_loader, eval_metrics=None, batch_end_callback=None):
    """"""
    # kvstore
    kv = mx.kvstore.create(args.kv_store)

    # logging
    head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s'
    if 'log_file' in args and args.log_file is not None:
        log_file = args.log_file
        log_dir = args.log_dir
        log_file_full_name = os.path.join(log_dir, log_file)
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)
        logger = logging.getLogger()
        handler = logging.FileHandler(log_file_full_name)
        formatter = logging.Formatter(head)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        logger.setLevel(logging.DEBUG)
        logger.info('start with arguments %s', args)
    else:
        logging.basicConfig(level=logging.DEBUG, format=head)
        logging.info('start with arguments %s', args)

    # load model
    model_prefix = args.model_prefix
    if model_prefix is not None:
        model_prefix += "-%d" % (kv.rank)
    model_args = {}
    if args.load_epoch is not None:
        assert model_prefix is not None
        tmp = mx.model.FeedForward.load(model_prefix, args.load_epoch)
        model_args = {'arg_params' : tmp.arg_params,
                      'aux_params' : tmp.aux_params,
                      'begin_epoch' : args.load_epoch}
    # save model
    save_model_prefix = args.save_model_prefix
    if save_model_prefix is None:
        save_model_prefix = model_prefix
    checkpoint = None if save_model_prefix is None else mx.callback.do_checkpoint(save_model_prefix)

    # data
    (train, val) = data_loader(args, kv)

    # train
    devs = mx.cpu() if args.gpus is None else [
        mx.gpu(int(i)) for i in args.gpus.split(',')]

    epoch_size = args.num_examples / args.batch_size

    if args.kv_store == 'dist_sync':
        epoch_size /= kv.num_workers
        model_args['epoch_size'] = epoch_size

    if 'lr_factor' in args and args.lr_factor < 1:
        model_args['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
            step=max(int(epoch_size * args.lr_factor_epoch), 1),
            factor=args.lr_factor)

    if 'clip_gradient' in args and args.clip_gradient is not None:
        model_args['clip_gradient'] = args.clip_gradient

    # disable kvstore for single device
    if 'local' in kv.type and (
            args.gpus is None or len(args.gpus.split(',')) is 1):
        kv = None

    mod = mx.mod.Module(network, context=devs)

    if eval_metrics is None:
        eval_metrics = ['accuracy']
        # TopKAccuracy only allows top_k > 1
        for top_k in [5, 10, 20]:
            eval_metrics.append(mx.metric.create('top_k_accuracy', top_k=top_k))

    if batch_end_callback is not None:
        if not isinstance(batch_end_callback, list):
            batch_end_callback = [batch_end_callback]
    else:
        batch_end_callback = []
    batch_end_callback.append(mx.callback.Speedometer(args.batch_size, 50))

    mod.fit(train_data=train, eval_metric=eval_metrics, eval_data=val, optimizer='sgd',
            optimizer_params={'learning_rate':args.lr, 'momentum': 0.9, 'wd': 0.00001},
            num_epoch=args.num_epochs, batch_end_callback=batch_end_callback,
            initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
            kvstore=kv, epoch_end_callback=checkpoint, **model_args)
123084
def run_mnist():
  """"""
  mnist_classifier = tf.estimator.Estimator(
      model_fn=model_fn,
      model_dir=FLAGS.model_dir)

  # Set up training and evaluation input functions.
  def train_input_fn():
    """Prepare data for training."""

    # When choosing shuffle buffer sizes, larger sizes result in better
    # randomness, while smaller sizes use less memory. MNIST is a small
    # enough dataset that we can easily shuffle the full epoch.
    ds = dataset.train(FLAGS.data_dir)
    ds_batched = ds.cache().shuffle(buffer_size=50000).batch(FLAGS.batch_size)

    # Iterate through the dataset a set number (`epochs_between_evals`) of times
    # during each training session.
    ds = ds_batched.repeat(FLAGS.epochs_between_evals)
    return ds

  def eval_input_fn():
    return dataset.test(FLAGS.data_dir).batch(
        FLAGS.batch_size).make_one_shot_iterator().get_next()

  # Train and evaluate model.
  for _ in range(FLAGS.train_epochs // FLAGS.epochs_between_evals):
    mnist_classifier.train(input_fn=train_input_fn, hooks=None)
    eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
    print("\nEvaluation results:\n\t%s\n" % eval_results)
1309049
def get_model(input_dim):
    """"""
    model = Sequential()
    model.add(Dense(100, input_dim=input_dim, activation='sigmoid'))
    model.add(Dense(1))
    model.compile(loss='mean_squared_error', optimizer='SGD')
    return model
773946
def learn(self, numEpochs, batchsize):
        """"""
        for epoch in range(numEpochs):
            print('epoch %d' % epoch)
            indexes = np.random.permutation(self.trainsize)
            for i in range(0, self.trainsize, batchsize):
                x = Variable(self.x_train[indexes[i: i + batchsize]])
                t = Variable(self.y_train[indexes[i: i + batchsize]])
                self.optimizer.update(self.model, x, t)
262985
def train(nr_iter, examples):
    '''
    '''
    model = AveragedPerceptron()
    for i in range(nr_iter):
        random.shuffle(examples)
        for features, class_ in examples:
            scores = model.predict(features)
            guess, score = max(scores.items(), key=lambda i: i[1])
            if guess != class_:
                model.update(class_, guess, features)
    model.average_weights()
    return model
837994
def train(self, data, label, batch_size):
        
        # pylint: disable=no-member
        sum_losses = 0
        len_losses = 0
        with autograd.record():
            losses = [self.loss_fn(self.net(X), Y) for X, Y in zip(data, label)]
        for loss in losses:
            sum_losses += mx.nd.array(loss).sum().asscalar()
            len_losses += len(loss)
            loss.backward()
        self.trainer.step(batch_size)
        return sum_losses, len_losses
909817
def convert(self, caffemodel_path, outmodel_path):
        """"""
        net_param = caffe_pb2.NetParameter()
        with open(caffemodel_path, 'rb') as caffe_model_file:
            net_param.ParseFromString(caffe_model_file.read())

        layers = net_param.layer
        self.layers = layers

        for idx, layer in enumerate(layers):
            layer_name = str(layer.name)

            if layer.blobs:

                # If this is a layer that has only weight and bias as parameter
                if layer.type == 'Convolution' or layer.type == 'InnerProduct' \
                        or layer.type == 'Deconvolution':

                    # Add weight and bias to the dictionary
                    self.add_arg_param('%s_weight' % layer_name, layer_index=idx, blob_index=0)
                    self.add_optional_arg_param('%s_bias' % layer_name, layer_index=idx,
                                                blob_index=1)

                elif layer.type == 'BatchNorm':

                    gamma_param_name = '%s_gamma' % layer_name
                    beta_param_name = '%s_beta' % layer_name

                    next_layer = layers[idx + 1]

                    if next_layer.type == 'Scale':
                        # If next layer is scale layer, get gamma and beta from there
                        self.add_arg_param(gamma_param_name, layer_index=idx+1, blob_index=0)
                        self.add_arg_param(beta_param_name, layer_index=idx+1, blob_index=1)

                    mean_param_name = '%s_moving_mean' % layer_name
                    var_param_name = '%s_moving_var' % layer_name

                    self.add_aux_param(mean_param_name, layer_index=idx, blob_index=0)
                    self.add_aux_param(var_param_name, layer_index=idx, blob_index=1)

                elif layer.type == 'Scale':

                    prev_layer = layers[idx - 1]

                    if prev_layer.type == 'BatchNorm':
                        continue
                    else:
                        # Use the naming convention used by CaffeOp
                        self.add_arg_param('%s_0_weight' % layer_name, layer_index=idx,
                                           blob_index=0)
                        self.add_optional_arg_param('%s_1_bias' % layer_name,
                                                    layer_index=idx, blob_index=1)

        mx.nd.save(outmodel_path, self.dict_param)
865627
def mnist_tutorial(nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
                   train_end=-1, test_end=-1, learning_rate=LEARNING_RATE):
  
  # Train a pytorch MNIST model
  torch_model = PytorchMnistModel()
  if torch.cuda.is_available():
    torch_model = torch_model.cuda()
  report = AccuracyReport()

  train_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=True, download=True,
                     transform=transforms.ToTensor()),
      batch_size=batch_size, shuffle=True)
  test_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=False, transform=transforms.ToTensor()),
      batch_size=batch_size)

  # Truncate the datasets so that our test run more quickly
  train_loader.dataset.train_data = train_loader.dataset.train_data[
      :train_end]
  test_loader.dataset.test_data = test_loader.dataset.test_data[:test_end]

  # Train our model
  optimizer = optim.Adam(torch_model.parameters(), lr=learning_rate)
  train_loss = []

  total = 0
  correct = 0
  step = 0
  for _epoch in range(nb_epochs):
    for xs, ys in train_loader:
      xs, ys = Variable(xs), Variable(ys)
      if torch.cuda.is_available():
        xs, ys = xs.cuda(), ys.cuda()
      optimizer.zero_grad()
      preds = torch_model(xs)
      loss = F.nll_loss(preds, ys)
      loss.backward()  # calc gradients
      train_loss.append(loss.data.item())
      optimizer.step()  # update gradients

      preds_np = preds.cpu().detach().numpy()
      correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
      total += train_loader.batch_size
      step += 1
      if total % 1000 == 0:
        acc = float(correct) / total
        print('[%s] Training accuracy: %.2f%%' % (step, acc * 100))
        total = 0
        correct = 0

  # Evaluate on clean data
  total = 0
  correct = 0
  for xs, ys in test_loader:
    xs, ys = Variable(xs), Variable(ys)
    if torch.cuda.is_available():
      xs, ys = xs.cuda(), ys.cuda()

    preds = torch_model(xs)
    preds_np = preds.cpu().detach().numpy()

    correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
    total += len(xs)

  acc = float(correct) / total
  report.clean_train_clean_eval = acc
  print('[%s] Clean accuracy: %.2f%%' % (step, acc * 100))

  # We use tf for evaluation on adversarial data
  sess = tf.Session()
  x_op = tf.placeholder(tf.float32, shape=(None, 1, 28, 28,))

  # Convert pytorch model to a tf_model and wrap it in cleverhans
  tf_model_fn = convert_pytorch_model_to_tf(torch_model)
  cleverhans_model = CallableModelWrapper(tf_model_fn, output_layer='logits')

  # Create an FGSM attack
  fgsm_op = FastGradientMethod(cleverhans_model, sess=sess)
  fgsm_params = {'eps': 0.3,
                 'clip_min': 0.,
                 'clip_max': 1.}
  adv_x_op = fgsm_op.generate(x_op, **fgsm_params)
  adv_preds_op = tf_model_fn(adv_x_op)

  # Run an evaluation of our model against fgsm
  total = 0
  correct = 0
  for xs, ys in test_loader:
    adv_preds = sess.run(adv_preds_op, feed_dict={x_op: xs})
    correct += (np.argmax(adv_preds, axis=1) == ys.cpu().detach().numpy()).sum()
    total += test_loader.batch_size

  acc = float(correct) / total
  print('Adv accuracy: {:.3f}'.format(acc * 100))
  report.clean_train_adv_eval = acc
  return report
730144
def train(dataset, nef, ndf, ngf, nc, batch_size, Z, lr, beta1, epsilon, ctx, check_point, g_dl_weight, output_path, checkpoint_path, data_path, activation,num_epoch, save_after_every, visualize_after_every, show_after_every):
    '''
    '''

    #encoder
    z_mu, z_lv, z = encoder(nef, Z, batch_size)
    symE = mx.sym.Group([z_mu, z_lv, z])

    #generator
    symG = generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim = Z, activation=activation )

    #discriminator
    h  = discriminator1(ndf)
    dloss  = discriminator2(ndf)
    symD1 = h
    symD2 = dloss


    # ==============data==============
    X_train, _ = get_data(data_path, activation)
    train_iter = mx.io.NDArrayIter(X_train, batch_size=batch_size, shuffle=True)
    rand_iter = RandIter(batch_size, Z)
    label = mx.nd.zeros((batch_size,), ctx=ctx)

    # =============module E=============
    modE = mx.mod.Module(symbol=symE, data_names=('data',), label_names=None, context=ctx)
    modE.bind(data_shapes=train_iter.provide_data)
    modE.init_params(initializer=mx.init.Normal(0.02))
    modE.init_optimizer(
        optimizer='adam',
        optimizer_params={
            'learning_rate': lr,
            'wd': 1e-6,
            'beta1': beta1,
            'epsilon': epsilon,
            'rescale_grad': (1.0/batch_size)
        })
    mods = [modE]

    # =============module G=============
    modG = mx.mod.Module(symbol=symG, data_names=('rand',), label_names=None, context=ctx)
    modG.bind(data_shapes=rand_iter.provide_data, inputs_need_grad=True)
    modG.init_params(initializer=mx.init.Normal(0.02))
    modG.init_optimizer(
        optimizer='adam',
        optimizer_params={
            'learning_rate': lr,
            'wd': 1e-6,
            'beta1': beta1,
            'epsilon': epsilon,
        })
    mods.append(modG)

    # =============module D=============
    modD1 = mx.mod.Module(symD1, label_names=[], context=ctx)
    modD2 = mx.mod.Module(symD2, label_names=('label',), context=ctx)
    modD = mx.mod.SequentialModule()
    modD.add(modD1).add(modD2, take_labels=True, auto_wiring=True)
    modD.bind(data_shapes=train_iter.provide_data,
              label_shapes=[('label', (batch_size,))],
              inputs_need_grad=True)
    modD.init_params(initializer=mx.init.Normal(0.02))
    modD.init_optimizer(
        optimizer='adam',
        optimizer_params={
            'learning_rate': lr,
            'wd': 1e-3,
            'beta1': beta1,
            'epsilon': epsilon,
            'rescale_grad': (1.0/batch_size)
        })
    mods.append(modD)


    # =============module DL=============
    symDL = DiscriminatorLayerLoss()
    modDL = mx.mod.Module(symbol=symDL, data_names=('data',), label_names=('label',), context=ctx)
    modDL.bind(data_shapes=[('data', (batch_size,nef * 4,4,4))], ################################################################################################################################ fix 512 here
              label_shapes=[('label', (batch_size,nef * 4,4,4))],
              inputs_need_grad=True)
    modDL.init_params(initializer=mx.init.Normal(0.02))
    modDL.init_optimizer(
        optimizer='adam',
        optimizer_params={
            'learning_rate': lr,
            'wd': 0.,
            'beta1': beta1,
            'epsilon': epsilon,
            'rescale_grad': (1.0/batch_size)
        })

    # =============module KL=============
    symKL = KLDivergenceLoss()
    modKL = mx.mod.Module(symbol=symKL, data_names=('data',), label_names=None, context=ctx)
    modKL.bind(data_shapes=[('data', (batch_size*2,Z))],
               inputs_need_grad=True)
    modKL.init_params(initializer=mx.init.Normal(0.02))
    modKL.init_optimizer(
        optimizer='adam',
        optimizer_params={
            'learning_rate': lr,
            'wd': 0.,
            'beta1': beta1,
            'epsilon': epsilon,
            'rescale_grad': (1.0/batch_size)
        })
    mods.append(modKL)

    def norm_stat(d):
        return mx.nd.norm(d)/np.sqrt(d.size)
    mon = mx.mon.Monitor(10, norm_stat, pattern=".*output|d1_backward_data", sort=True)
    mon = None
    if mon is not None:
        for mod in mods:
            pass

    def facc(label, pred):
        '''calculating prediction accuracy
        '''
        pred = pred.ravel()
        label = label.ravel()
        return ((pred > 0.5) == label).mean()

    def fentropy(label, pred):
        '''calculating binary cross-entropy loss
        '''
        pred = pred.ravel()
        label = label.ravel()
        return -(label*np.log(pred+1e-12) + (1.-label)*np.log(1.-pred+1e-12)).mean()

    def kldivergence(label, pred):
        '''calculating KL divergence loss
        '''
        mean, log_var = np.split(pred, 2, axis=0)
        var = np.exp(log_var)
        KLLoss = -0.5 * np.sum(1 + log_var - np.power(mean, 2) - var)
        KLLoss = KLLoss / nElements
        return KLLoss

    mG = mx.metric.CustomMetric(fentropy)
    mD = mx.metric.CustomMetric(fentropy)
    mE = mx.metric.CustomMetric(kldivergence)
    mACC = mx.metric.CustomMetric(facc)

    print('Training...')
    stamp =  datetime.now().strftime('%Y_%m_%d-%H_%M')

    # =============train===============
    for epoch in range(num_epoch):
        train_iter.reset()
        for t, batch in enumerate(train_iter):

            rbatch = rand_iter.next()

            if mon is not None:
                mon.tic()

            modG.forward(rbatch, is_train=True)
            outG = modG.get_outputs()

            # update discriminator on fake
            label[:] = 0
            modD.forward(mx.io.DataBatch(outG, [label]), is_train=True)
            modD.backward()
            gradD11 = [[grad.copyto(grad.context) for grad in grads] for grads in modD1._exec_group.grad_arrays]
            gradD12 = [[grad.copyto(grad.context) for grad in grads] for grads in modD2._exec_group.grad_arrays]

            modD.update_metric(mD, [label])
            modD.update_metric(mACC, [label])


            #update discriminator on decoded
            modE.forward(batch, is_train=True)
            mu, lv, z = modE.get_outputs()
            z = z.reshape((batch_size, Z, 1, 1))
            sample = mx.io.DataBatch([z], label=None, provide_data = [('rand', (batch_size, Z, 1, 1))])
            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()
            label[:] = 0
            modD.forward(mx.io.DataBatch(xz, [label]), is_train=True)
            modD.backward()

            #modD.update()
            gradD21 = [[grad.copyto(grad.context) for grad in grads] for grads in modD1._exec_group.grad_arrays]
            gradD22 = [[grad.copyto(grad.context) for grad in grads] for grads in modD2._exec_group.grad_arrays]
            modD.update_metric(mD, [label])
            modD.update_metric(mACC, [label])

            # update discriminator on real
            label[:] = 1
            batch.label = [label]
            modD.forward(batch, is_train=True)
            lx = [out.copyto(out.context) for out in modD1.get_outputs()]
            modD.backward()
            for gradsr, gradsf, gradsd in zip(modD1._exec_group.grad_arrays, gradD11, gradD21):
                for gradr, gradf, gradd in zip(gradsr, gradsf, gradsd):
                    gradr += 0.5 * (gradf + gradd)
            for gradsr, gradsf, gradsd in zip(modD2._exec_group.grad_arrays, gradD12, gradD22):
                for gradr, gradf, gradd in zip(gradsr, gradsf, gradsd):
                    gradr += 0.5 * (gradf + gradd)

            modD.update()
            modD.update_metric(mD, [label])
            modD.update_metric(mACC, [label])

            modG.forward(rbatch, is_train=True)
            outG = modG.get_outputs()
            label[:] = 1
            modD.forward(mx.io.DataBatch(outG, [label]), is_train=True)
            modD.backward()
            diffD = modD1.get_input_grads()
            modG.backward(diffD)
            gradG1 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays]
            mG.update([label], modD.get_outputs())

            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()
            label[:] = 1
            modD.forward(mx.io.DataBatch(xz, [label]), is_train=True)
            modD.backward()
            diffD = modD1.get_input_grads()
            modG.backward(diffD)
            gradG2 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays]
            mG.update([label], modD.get_outputs())

            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()
            modD1.forward(mx.io.DataBatch(xz, []), is_train=True)
            outD1 = modD1.get_outputs()
            modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True)
            modDL.backward()
            dlGrad = modDL.get_input_grads()
            modD1.backward(dlGrad)
            diffD = modD1.get_input_grads()
            modG.backward(diffD)

            for grads, gradsG1, gradsG2 in zip(modG._exec_group.grad_arrays, gradG1, gradG2):
                for grad, gradg1, gradg2 in zip(grads, gradsG1, gradsG2):
                    grad = g_dl_weight * grad + 0.5 * (gradg1 + gradg2)

            modG.update()
            mG.update([label], modD.get_outputs())

            modG.forward(rbatch, is_train=True)
            outG = modG.get_outputs()
            label[:] = 1
            modD.forward(mx.io.DataBatch(outG, [label]), is_train=True)
            modD.backward()
            diffD = modD1.get_input_grads()
            modG.backward(diffD)
            gradG1 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays]
            mG.update([label], modD.get_outputs())

            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()
            label[:] = 1
            modD.forward(mx.io.DataBatch(xz, [label]), is_train=True)
            modD.backward()
            diffD = modD1.get_input_grads()
            modG.backward(diffD)
            gradG2 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays]
            mG.update([label], modD.get_outputs())

            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()
            modD1.forward(mx.io.DataBatch(xz, []), is_train=True)
            outD1 = modD1.get_outputs()
            modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True)
            modDL.backward()
            dlGrad = modDL.get_input_grads()
            modD1.backward(dlGrad)
            diffD = modD1.get_input_grads()
            modG.backward(diffD)

            for grads, gradsG1, gradsG2 in zip(modG._exec_group.grad_arrays, gradG1, gradG2):
                for grad, gradg1, gradg2 in zip(grads, gradsG1, gradsG2):
                    grad = g_dl_weight * grad + 0.5 * (gradg1 + gradg2)

            modG.update()
            mG.update([label], modD.get_outputs())

            modG.forward(sample, is_train=True)
            xz = modG.get_outputs()

            #update generator
            modD1.forward(mx.io.DataBatch(xz, []), is_train=True)
            outD1 = modD1.get_outputs()
            modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True)
            DLloss = modDL.get_outputs()
            modDL.backward()
            dlGrad = modDL.get_input_grads()
            modD1.backward(dlGrad)
            diffD = modD1.get_input_grads()
            modG.backward(diffD)
            #update encoder
            nElements = batch_size
            modKL.forward(mx.io.DataBatch([mx.ndarray.concat(mu,lv, dim=0)]), is_train=True)
            KLloss = modKL.get_outputs()
            modKL.backward()
            gradKLLoss = modKL.get_input_grads()
            diffG = modG.get_input_grads()
            diffG = diffG[0].reshape((batch_size, Z))
            modE.backward(mx.ndarray.split(gradKLLoss[0], num_outputs=2, axis=0) + [diffG])
            modE.update()
            pred = mx.ndarray.concat(mu,lv, dim=0)
            mE.update([pred], [pred])
            if mon is not None:
                mon.toc_print()

            t += 1
            if t % show_after_every == 0:
                print('epoch:', epoch, 'iter:', t, 'metric:', mACC.get(), mG.get(), mD.get(), mE.get(), KLloss[0].asnumpy(), DLloss[0].asnumpy())
                mACC.reset()
                mG.reset()
                mD.reset()
                mE.reset()

            if epoch % visualize_after_every == 0:
                visual(output_path +'gout'+str(epoch), outG[0].asnumpy(), activation)
                visual(output_path + 'data'+str(epoch), batch.data[0].asnumpy(), activation)

        if check_point and epoch % save_after_every == 0:
            print('Saving...')
            modG.save_params(checkpoint_path + '/%s_G-%04d.params'%(dataset, epoch))
            modD.save_params(checkpoint_path + '/%s_D-%04d.params'%(dataset, epoch))
            modE.save_params(checkpoint_path + '/%s_E-%04d.params'%(dataset, epoch))
291809
def train(self, training_set, iterations=500):
        """"""
        if len(training_set) > 2:
            self.__X = np.matrix([example[0] for example in training_set])
            if self.__num_labels == 1:
                self.__y = np.matrix([example[1] for example in training_set]).reshape((-1, 1))
            else:
                eye = np.eye(self.__num_labels)
                self.__y = np.matrix([eye[example[1]] for example in training_set])
        else:
            self.__X = np.matrix(training_set[0])
            if self.__num_labels == 1:
                self.__y = np.matrix(training_set[1]).reshape((-1, 1))
            else:
                eye = np.eye(self.__num_labels)
                self.__y = np.matrix([eye[index] for sublist in training_set[1] for index in sublist])
        self.__m = self.__X.shape[0]
        self.__input_layer_size = self.__X.shape[1]
        self.__sizes = [self.__input_layer_size]
        self.__sizes.extend(self.__hidden_layers)
        self.__sizes.append(self.__num_labels)
        initial_theta = []
        for count in range(len(self.__sizes) - 1):
            epsilon = np.sqrt(6) / np.sqrt(self.__sizes[count]+self.__sizes[count+1])
            initial_theta.append(np.random.rand(self.__sizes[count+1],self.__sizes[count]+1)*2*epsilon-epsilon)
        initial_theta = self.__unroll(initial_theta)
        self.__thetas = self.__roll(fmin_bfgs(self.__cost_function, initial_theta, fprime=self.__cost_grad_function, maxiter=iterations))
378278
def train(self, sentences, model):
        
        trainer = pycrfsuite.Trainer(verbose=True)
        trainer.set_params(self.params)
        for sentence in sentences:
            tokens, labels = zip(*sentence)
            features = [self._get_features(tokens, i) for i in range(len(tokens))]
            trainer.append(features, labels)
        trainer.train(model)
        self.load(model)
762718
def _train_lbfgs(self, X_feat_train, X_seq_train, y_train,
                     X_feat_valid, X_seq_valid, y_valid,
                     graph, var, other_var,
                     early_stop_patience=None,
                     n_cores=3):
        
        tic = time.time()
        # take out the parameters for conveience
        n_epochs = self._param["n_epochs"]
        print_every = self._param["print_every"]
        step_size = self._param["step_size"]
        num_steps = n_epochs

        print('Number of epochs:', n_epochs)
        # print("Number of steps per epoch:", num_steps)
        # print("Number of total steps:", num_steps * n_epochs)

        # move into the graph and start the model
        loss_history = []
        train_acc_vec = []
        valid_acc_vec = []
        step_history = []

        with tf.Session(graph=graph, config=tf.ConfigProto(
                use_per_session_threads=True,
                inter_op_parallelism_threads=n_cores,
                intra_op_parallelism_threads=n_cores)) as sess:

            sess.run(other_var["init"])

            best_performance = None
            best_performance_epoch = 0
            for step in range(n_epochs):
                # run the model (sess.run)
                # compute the optimizer, loss and train_prediction in the graph
                # save the last two as l and predictions

                # put thet data into TF form:
                feed_dict = {other_var["tf_X_seq"]: X_seq_train, other_var["tf_y"]: y_train,
                             other_var["tf_X_feat"]: X_feat_train,
                             other_var["tf_step_size"]: step_size}

                # run the optimizer
                # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/opt/python/training/external_optimizer.py#L115
                other_var["optimizer"].minimize(sess, feed_dict=feed_dict)
                l = sess.run(other_var["loss"], feed_dict=feed_dict)
                loss_history.append(l)  # keep storing the full loss history

                # sometimes print the actual training prediction (l)
                if (step % print_every == 0):
                    train_accuracy = self._accuracy_in_session(sess, other_var,
                                                               X_feat_train, X_seq_train, y_train)
                    valid_accuracy = self._accuracy_in_session(sess, other_var,
                                                               X_feat_valid, X_seq_valid, y_valid)
                    # append the prediction accuracies
                    train_acc_vec.append(train_accuracy)
                    valid_acc_vec.append(valid_accuracy)
                    step_history.append(step / num_steps)
                    print('Step %4d: loss %f, train mse: %f, validation mse: %f' %
                          (step, l, train_accuracy, valid_accuracy))
                    # check if this is the best accuracy
                    if best_performance is None or valid_accuracy <= best_performance:
                        best_performance = valid_accuracy
                        best_performance_epoch = step

                if early_stop_patience is not None and step > best_performance_epoch + early_stop_patience:
                    print("Early stopping. best_performance_epoch: %d, best_performance: %f" %
                          (best_performance_epoch, best_performance))
                    break

            # get the test accuracies
            train_accuracy_final = self._accuracy_in_session(sess, other_var,
                                                             X_feat_train, X_seq_train, y_train)
            valid_accuracy_final = self._accuracy_in_session(sess, other_var,
                                                             X_feat_valid, X_seq_valid, y_valid)
            print('Validation accuracy final: %f' % valid_accuracy_final)

            # store the fitted weights
            var_res = self._get_var_res_sess(sess, var)

            # store also the quasi splines fit

            if self._param["n_splines"] is not None:
                self._splines["quasi_X"] = [self._predict_in_session(sess, other_var,
                                                                     X_feat_train[i:(i + 1)],
                                                                     X_seq_train[i:(i + 1)],
                                                                     variable="spline_quasi_X")
                                            for i in range(X_feat_train.shape[0])]
                # transform into the appropriate form
                self._splines["quasi_X"] = np.concatenate([x[0][np.newaxis] for x in self._splines["quasi_X"]])

            accuracy = {
                "loss_history": np.array(loss_history),
                "step_history": np.array(step_history),
                "train_acc_history": np.array(train_acc_vec),
                "val_acc_history": np.array(valid_acc_vec),
                "train_acc_final": train_accuracy_final,
                "val_acc_final": valid_accuracy_final,
                "best_val_acc": best_performance,
                "best_val_acc_epoch": best_performance_epoch,
                "test_acc_final": None,  # test_accuracy_final,
                "y_test": None,  # y_test,
                "y_test_prediction": None,  # test_prediction.eval(),
                "id_vec_test": None  # id_vec_test
            }
            self._accuracy = accuracy

        toc = time.time()
        exec_time = toc - tic
        self._exec_time = exec_time
        print('That took %fs' % exec_time)
        # weights = {"motif_base_weights": motif_base_weights,
        #            "motif_weights": motif_weights,
        #            "motif_bias": motif_bias,
        #            "final_bias": final_bias,
        #            "feature_weights": feature_weights,
        #            "spline_pred": spline_pred
        #            }
        return var_res
173771
def train_epoch(model:nn.Module, dl:DataLoader, opt:optim.Optimizer, loss_func:LossFunction)->None:
    ""
    model.train()
    for xb,yb in dl:
        loss = loss_func(model(xb), yb)
        loss.backward()
        opt.step()
        opt.zero_grad()
1342652
def build_model():
  
  # training data
  train_data_node = tf.placeholder(tf.float32,shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  train_labels_node = tf.placeholder(tf.float32,shape=(BATCH_SIZE, NUM_LABELS))

  validation_data_node= tf.placeholder(tf.float32,shape=(VALIDATION_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  test_data_node=tf.placeholder(tf.float32,shape=(TEST_SIZE,IMAGE_SIZE,IMAGE_SIZE,NUM_CHANNELS))
  # validation dataset held in a single constant node
  # validation_data_node = tf.constant(validation_data)
  # test_data_node = tf.constant(test_data)

  # LEARNABLE WEIGHT NODES SHARED BETWEEN
  conv1_weights = tf.Variable(tf.truncated_normal([5, 5, NUM_CHANNELS, 32],stddev=0.1,seed=SEED))
  conv1_biases = tf.Variable(tf.zeros([32]))
  conv2_weights = tf.Variable(tf.truncated_normal([5, 5, 32, 64],stddev=0.1,seed=SEED))
  conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))
  fc1_weights = tf.Variable(tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],stddev=0.1,seed=SEED))
  fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))
  fc2_weights = tf.Variable(tf.truncated_normal([512, NUM_LABELS],stddev=0.1,seed=SEED))
  fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))

  # LENET
  def build_lenet(data,train=False):
    # subroutine for wiring up nodes and weights to training and evaluation LeNets
    conv1 = tf.nn.conv2d(data,conv1_weights,strides=[1, 1, 1, 1],padding='SAME')
    relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
    pool1 = tf.nn.max_pool(relu1,ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME')
    conv2 = tf.nn.conv2d(pool1,conv2_weights,strides=[1, 1, 1, 1],padding='SAME')
    relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
    pool2 = tf.nn.max_pool(relu2,ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME')
    # Reshape the feature map cuboid into a 2D matrix to feed it to the
    # fully connected layers.
    pool_shape = pool2.get_shape().as_list()
    reshape = tf.reshape(pool2,[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
    fc1 = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
    # Add a 50% dropout during training only. Dropout also scales
    # activations such that no rescaling is needed at evaluation time.
    if train:
      fc1 = tf.nn.dropout(fc1, 0.5, seed=SEED)
      # append summary ops to train
      _activation_summary(conv1)
      _activation_summary(fc1)

    fc2 = tf.matmul(fc1, fc2_weights) + fc2_biases
    return fc2

  # TRAINING LOSS / REGULARIZATION NODES
  logits = build_lenet(train_data_node, True)
  loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, train_labels_node))

  tf.scalar_summary(loss.op.name,loss)

  regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) + tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
  # Add the regularization term to the loss.
  loss += 5e-4 * regularizers

  # OPTIMIZER NODES
  batch = tf.Variable(0)
  # Decay once per epoch, using an exponential schedule starting at 0.01.
  learning_rate = tf.train.exponential_decay(
    0.01,                # Base learning rate.
    batch * BATCH_SIZE,  # Current index into the dataset.
    TRAIN_SIZE,          # Decay step.
    0.95,                # Decay rate.
    staircase=True)
  # Use simple momentum for the optimization.
  optimizer = tf.train.MomentumOptimizer(learning_rate,0.9).minimize(loss,global_step=batch)

  # # Predictions for the minibatch, validation set and test set.
  train_prediction = tf.nn.softmax(logits)
  # # We'll compute them only once in a while by calling their {eval()} method.
  validation_prediction = tf.nn.softmax(build_lenet(validation_data_node))
  test_prediction = tf.nn.softmax(build_lenet(test_data_node))

  summaries=tf.merge_all_summaries()

  # return input nodes and output nodes
  return (train_data_node,
    train_labels_node,
    validation_data_node,
    test_data_node,
    train_prediction,
    validation_prediction,
    test_prediction,
    conv1_weights,
    conv2_weights,
    fc1_weights,
    fc2_weights,
    optimizer,
    loss,
    learning_rate,
    summaries)
586569
def train_crf(ctx, input, output, clusters):
    """"""
    click.echo('chemdataextractor.crf.train')
    sentences = []
    for line in input:
        sentence = []
        for t in line.split():
            token, tag, iob = t.rsplit('/', 2)
            sentence.append(((token, tag), iob))
        if sentence:
            sentences.append(sentence)

    tagger = CrfCemTagger(clusters=clusters)
    tagger.train(sentences, output)
317884
def train(X_train, y_train, **kwargs):
    '''
    
    '''
    crf = Trainer()
    crf.set_params({
        'c1': 1.0,
        'c2': 0.001,
        'max_iterations': 200,
        'feature.possible_transitions': True,
    })

    for xseq, yseq in zip(X_train, y_train):
        crf.append(xseq, yseq)
    crf.train(TOKENIZATION_MODEL_PATH)
    return crf
518775
@Override
    public void trainOnInstanceImpl(Instance instance) {

        if (m_weights == null) {
            m_weights = new DoubleVector();
            m_velocity = new DoubleVector();
            m_bias = 0;

            m_weights.setValue(instance.numAttributes(), 0);
            m_velocity.setValue(instance.numAttributes(), 0);
        }

        if (instance.classIsMissing()) {
            return;
        }
   
        double z = dotProd(instance, m_weights, instance.classIndex()) + m_bias;

        double y;
        double dldz;

        if (instance.classAttribute().isNominal()) {
            y = (instance.classValue() == 0) ? 0 : 1;
            
            if (m_loss == LOGLOSS) {
                double yhat = 1.0 / (1.0 + Math.exp(-z));
                dldz = (yhat - y);
            }
            else {
                y = y * 2 - 1;
                
                if(y * z < 1.0)
                {
                    dldz = -y;
                }
                else
                {
                    dldz = 0;
                }
            }
        }
        else {
            y = instance.classValue();
            dldz = z - y;
        }

        int n = instance.numValues();
        DoubleVector gradients = new DoubleVector();
        gradients.setValue(instance.numAttributes(), 0);

        for(int i = 0; i < n; i++)
        {
            int idx = instance.index(i);
            gradients.setValue(idx, instance.valueSparse(i) * dldz + (m_lambda / (m_t + m_epsilon)) * m_weights.getValue(idx));
        }

        //Weight update for the bias
        double biasGradient = dldz;
        m_biasVelocity += biasGradient * biasGradient;
        m_bias -= (m_learningRate / (Math.sqrt(m_biasVelocity) + m_epsilon)) * biasGradient;

        for(int i = 0; i < m_weights.numValues(); i++) {
            //Weight update
            double g = gradients.getValue(i);
            m_velocity.addToValue(i, g * g);
            m_weights.addToValue(i, -(m_learningRate / (Math.sqrt(m_velocity.getValue(i)) + m_epsilon)) * g);
        }

        m_t += 1.0;
    }