Caffe2 implementation uses XavierFill, which in fact
        corresponds to kaiming_uniform_ in PyTorch
775106
def make_fc(dim_in, hidden_dim, use_gn=False):
    '''
        
    '''
    if use_gn:
        fc = nn.Linear(dim_in, hidden_dim, bias=False)
        nn.init.kaiming_uniform_(fc.weight, a=1)
        return nn.Sequential(fc, group_norm(hidden_dim))
    fc = nn.Linear(dim_in, hidden_dim)
    nn.init.kaiming_uniform_(fc.weight, a=1)
    nn.init.constant_(fc.bias, 0)
    return fc
309275
def init_from_fcnxs(ctx, fcnxs_symbol, fcnxs_args_from, fcnxs_auxs_from):
    
    fcnxs_args = fcnxs_args_from.copy()
    fcnxs_auxs = fcnxs_auxs_from.copy()
    for k,v in fcnxs_args.items():
        if(v.context != ctx):
            fcnxs_args[k] = mx.nd.zeros(v.shape, ctx)
            v.copyto(fcnxs_args[k])
    for k,v in fcnxs_auxs.items():
        if(v.context != ctx):
            fcnxs_auxs[k] = mx.nd.zeros(v.shape, ctx)
            v.copyto(fcnxs_auxs[k])
    data_shape=(1,3,500,500)
    arg_names = fcnxs_symbol.list_arguments()
    arg_shapes, _, _ = fcnxs_symbol.infer_shape(data=data_shape)
    rest_params = {}
    deconv_params = {}
    # this is fcn8s init from fcn16s
    if 'score_pool3_weight' in arg_names:
        rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes)
            if x[0] in ['score_pool3_bias', 'score_pool3_weight']])
        deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \
            in ["bigscore_weight", 'score4_weight']])
    # this is fcn16s init from fcn32s
    elif 'score_pool4_weight' in arg_names:
        rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes)
            if x[0] in ['score_pool4_weight', 'score_pool4_bias']])
        deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \
            in ["bigscore_weight", 'score2_weight']])
    # this is fcn32s init
    else:
        logging.error("you are init the fcn32s model, so you should use init_from_vgg16()")
        sys.exit()
    fcnxs_args.update(rest_params)
    for k, v in deconv_params.items():
        filt = upsample_filt(v[3])
        initw = np.zeros(v)
        initw[range(v[0]), range(v[1]), :, :] = filt  # becareful here is the slice assing
        fcnxs_args[k] = mx.nd.array(initw, ctx)
    return fcnxs_args, fcnxs_auxs
87788
void defaultXGBoosts(boolean emulateLightGBM) {
    Algo algo = Algo.XGBoost;
    WorkAllocations.Work work = workAllocations.getAllocation(algo, JobType.ModelBuild);
    if (work == null) return;

    XGBoostParameters xgBoostParameters = new XGBoostParameters();
    setCommonModelBuilderParams(xgBoostParameters);

    Job xgBoostJob;
    Key<Model> key;

    if (emulateLightGBM) {
      xgBoostParameters._tree_method = XGBoostParameters.TreeMethod.hist;
      xgBoostParameters._grow_policy = XGBoostParameters.GrowPolicy.lossguide;
    }

    // setDistribution: no way to identify gaussian, poisson, laplace? using descriptive statistics?
    xgBoostParameters._distribution = getResponseColumn().isBinary() && !(getResponseColumn().isNumeric()) ? DistributionFamily.bernoulli
                    : getResponseColumn().isCategorical() ? DistributionFamily.multinomial
                    : DistributionFamily.AUTO;

    xgBoostParameters._score_tree_interval = 5;
    xgBoostParameters._stopping_rounds = 5;
//    xgBoostParameters._stopping_tolerance = Math.min(1e-2, RandomDiscreteValueSearchCriteria.default_stopping_tolerance_for_frame(this.trainingFrame));

    xgBoostParameters._ntrees = 10000;
    xgBoostParameters._learn_rate = 0.05;
//    xgBoostParameters._min_split_improvement = 0.01f;

    //XGB 1 (medium depth)
    xgBoostParameters._max_depth = 10;
    xgBoostParameters._min_rows = 5;
    xgBoostParameters._sample_rate = 0.6;
    xgBoostParameters._col_sample_rate = 0.8;
    xgBoostParameters._col_sample_rate_per_tree = 0.8;

    if (emulateLightGBM) {
      xgBoostParameters._max_leaves = 1 << xgBoostParameters._max_depth;
      xgBoostParameters._max_depth = xgBoostParameters._max_depth * 2;
//      xgBoostParameters._min_data_in_leaf = (float) xgBoostParameters._min_rows;
      xgBoostParameters._min_sum_hessian_in_leaf = (float) xgBoostParameters._min_rows;
    }

    key = modelKey(algo.name());
    xgBoostJob = trainModel(key, work, xgBoostParameters);
    pollAndUpdateProgress(Stage.ModelTraining,  key.toString(), work, this.job(), xgBoostJob);

    //XGB 2 (deep)
    xgBoostParameters._max_depth = 20;
    xgBoostParameters._min_rows = 10;
    xgBoostParameters._sample_rate = 0.6;
    xgBoostParameters._col_sample_rate = 0.8;
    xgBoostParameters._col_sample_rate_per_tree = 0.8;

    if (emulateLightGBM) {
      xgBoostParameters._max_leaves = 1 << xgBoostParameters._max_depth;
      xgBoostParameters._max_depth = xgBoostParameters._max_depth * 2;
//      xgBoostParameters._min_data_in_leaf = (float) xgBoostParameters._min_rows;
      xgBoostParameters._min_sum_hessian_in_leaf = (float) xgBoostParameters._min_rows;
    }

    key = modelKey(algo.name());
    xgBoostJob = trainModel(key, work, xgBoostParameters);
    pollAndUpdateProgress(Stage.ModelTraining,  key.toString(), work, this.job(), xgBoostJob);

    //XGB 3 (shallow)
    xgBoostParameters._max_depth = 5;
    xgBoostParameters._min_rows = 3;
    xgBoostParameters._sample_rate = 0.8;
    xgBoostParameters._col_sample_rate = 0.8;
    xgBoostParameters._col_sample_rate_per_tree = 0.8;

    if (emulateLightGBM) {
      xgBoostParameters._max_leaves = 1 << xgBoostParameters._max_depth;
      xgBoostParameters._max_depth = xgBoostParameters._max_depth * 2;
//      xgBoostParameters._min_data_in_leaf = (float) xgBoostParameters._min_rows;
      xgBoostParameters._min_sum_hessian_in_leaf = (float) xgBoostParameters._min_rows;
    }

    key = modelKey(algo.name());
    xgBoostJob = trainModel(key, work, xgBoostParameters);
    pollAndUpdateProgress(Stage.ModelTraining,  key.toString(), work, this.job(), xgBoostJob);
  }
865627
def mnist_tutorial(nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
                   train_end=-1, test_end=-1, learning_rate=LEARNING_RATE):
  
  # Train a pytorch MNIST model
  torch_model = PytorchMnistModel()
  if torch.cuda.is_available():
    torch_model = torch_model.cuda()
  report = AccuracyReport()

  train_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=True, download=True,
                     transform=transforms.ToTensor()),
      batch_size=batch_size, shuffle=True)
  test_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=False, transform=transforms.ToTensor()),
      batch_size=batch_size)

  # Truncate the datasets so that our test run more quickly
  train_loader.dataset.train_data = train_loader.dataset.train_data[
      :train_end]
  test_loader.dataset.test_data = test_loader.dataset.test_data[:test_end]

  # Train our model
  optimizer = optim.Adam(torch_model.parameters(), lr=learning_rate)
  train_loss = []

  total = 0
  correct = 0
  step = 0
  for _epoch in range(nb_epochs):
    for xs, ys in train_loader:
      xs, ys = Variable(xs), Variable(ys)
      if torch.cuda.is_available():
        xs, ys = xs.cuda(), ys.cuda()
      optimizer.zero_grad()
      preds = torch_model(xs)
      loss = F.nll_loss(preds, ys)
      loss.backward()  # calc gradients
      train_loss.append(loss.data.item())
      optimizer.step()  # update gradients

      preds_np = preds.cpu().detach().numpy()
      correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
      total += train_loader.batch_size
      step += 1
      if total % 1000 == 0:
        acc = float(correct) / total
        print('[%s] Training accuracy: %.2f%%' % (step, acc * 100))
        total = 0
        correct = 0

  # Evaluate on clean data
  total = 0
  correct = 0
  for xs, ys in test_loader:
    xs, ys = Variable(xs), Variable(ys)
    if torch.cuda.is_available():
      xs, ys = xs.cuda(), ys.cuda()

    preds = torch_model(xs)
    preds_np = preds.cpu().detach().numpy()

    correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
    total += len(xs)

  acc = float(correct) / total
  report.clean_train_clean_eval = acc
  print('[%s] Clean accuracy: %.2f%%' % (step, acc * 100))

  # We use tf for evaluation on adversarial data
  sess = tf.Session()
  x_op = tf.placeholder(tf.float32, shape=(None, 1, 28, 28,))

  # Convert pytorch model to a tf_model and wrap it in cleverhans
  tf_model_fn = convert_pytorch_model_to_tf(torch_model)
  cleverhans_model = CallableModelWrapper(tf_model_fn, output_layer='logits')

  # Create an FGSM attack
  fgsm_op = FastGradientMethod(cleverhans_model, sess=sess)
  fgsm_params = {'eps': 0.3,
                 'clip_min': 0.,
                 'clip_max': 1.}
  adv_x_op = fgsm_op.generate(x_op, **fgsm_params)
  adv_preds_op = tf_model_fn(adv_x_op)

  # Run an evaluation of our model against fgsm
  total = 0
  correct = 0
  for xs, ys in test_loader:
    adv_preds = sess.run(adv_preds_op, feed_dict={x_op: xs})
    correct += (np.argmax(adv_preds, axis=1) == ys.cpu().detach().numpy()).sum()
    total += test_loader.batch_size

  acc = float(correct) / total
  print('Adv accuracy: {:.3f}'.format(acc * 100))
  report.clean_train_adv_eval = acc
  return report
1147017
function generateCFIs(cfiBase, elements, needOffset) {

    const cfiList = [];

    for (const key in elements) {

        const cfiParts = [];

        const textNode = elements[key].textNode;
        var child = textNode.parent();
        const childContents = child.contents();


        var textNodeIndex = childContents.index(textNode) + 1;

        // "mixed content" context
        // the first chunk is located before the first child element
        // <p><span></span>text</p>
        if (childContents.first()[0].type === "tag") {
            textNodeIndex += 1;
        }

        var parent = child.parent();
        while (parent[0]) {
            const index = child.index(),
                inOff = (needOffset && parent[0].name === 'body'),
                id = child.attr('id'),
                idSelector = id ? '[' + id + ']' : '',
                part = ((index + 1) * 2 + (inOff ? 2 : 0)) + idSelector;

            cfiParts.unshift(part);

            child = parent;
            parent = child.parent();
        }
        const startOffset = elements[key].range.startOffset,
            endOffset = elements[key].range.endOffset;

        const inlinePath = ',/' + textNodeIndex + ':';
        const cfi = cfiBase + '/' + cfiParts.join('/') + inlinePath + startOffset + inlinePath + endOffset;

        cfiList.push({ cfi, excerpt: elements[key].excerpt });
    }
    return cfiList;
}
1049851
def set(self, opts, popsize=None, ccovfac=1, verbose=True):
        

        alpha_cc = 1.0  # cc-correction for mueff, was zero before

        def conedf(df, mu, N):
            """used for computing separable learning rate"""
            return 1. / (df + 2.*sqrt(df) + float(mu) / N)

        def cmudf(df, mu, alphamu):
            """used for computing separable learning rate"""
            return (alphamu + mu - 2. + 1. / mu) / (df + 4.*sqrt(df) + mu / 2.)

        sp = self
        N = sp.N
        if popsize:
            opts.evalall({'N':N, 'popsize':popsize})
        else:
            popsize = opts.evalall({'N':N})['popsize']  # the default popsize is computed in CMAOptions()
        ## meta_parameters.lambda_exponent == 0.0
        popsize = int(popsize + N** 0.0 - 1)
        sp.popsize = popsize
        if opts['CMA_mirrors'] < 0.5:
            sp.lam_mirr = int(0.5 + opts['CMA_mirrors'] * popsize)
        elif opts['CMA_mirrors'] > 1:
            sp.lam_mirr = int(0.5 + opts['CMA_mirrors'])
        else:
            sp.lam_mirr = int(0.5 + 0.16 * min((popsize, 2 * N + 2)) + 0.29)  # 0.158650... * popsize is optimal
            # lam = arange(2,22)
            # mirr = 0.16 + 0.29/lam
            # print(lam); print([int(0.5 + l) for l in mirr*lam])
            # [ 2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21]
            # [1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4]
        ## meta_parameters.parent_fraction == 0.5
        sp.mu_f = 0.5 * sp.popsize  # float value of mu
        if opts['CMA_mu'] is not None:
            sp.mu_f = opts['CMA_mu']
        sp.mu = int(sp.mu_f + 0.499999)  # round down for x.5
        sp.mu = max((sp.mu, 1))
        # in principle we have mu_opt = popsize/2 + lam_mirr/2,
        # which means in particular weights should only be negative for q > 0.5+mirr_frac/2
        if sp.mu > sp.popsize - 2 * sp.lam_mirr + 1:
            _print_warning("pairwise selection is not implemented, therefore " +
                  " mu = %d > %d = %d - 2*%d + 1 = popsize - 2*mirr + 1 can produce a bias" % (
                    sp.mu, sp.popsize - 2 * sp.lam_mirr + 1, sp.popsize, sp.lam_mirr))
        if sp.lam_mirr > sp.popsize // 2:
            raise _Error("fraction of mirrors in the population as read from option CMA_mirrors cannot be larger 0.5, " +
                         "theoretically optimal is 0.159")
        sp.weights = log(max([sp.mu, sp.popsize / 2.0]) + 0.5) - log(1 + np.arange(sp.mu))
        sp.weights /= sum(sp.weights)
        sp.mueff = 1 / sum(sp.weights**2)
        # TODO: this will disappear, as it is done in class CMAAdaptSigmaCSA
        ## meta_parameters.cs_exponent == 1.0
        b = 1.0
        ## meta_parameters.cs_multiplier == 1.0
        sp.cs = 1.0 * (sp.mueff + 2)**b / (N + (sp.mueff + 3)**b)  # TODO: this doesn't change dependency of dimension
        # sp.cs = (sp.mueff + 2) / (N + 1.5*sp.mueff + 1)
        ## meta_parameters.cc_exponent == 1.0
        b = 1.0
        ## meta_parameters.cc_multiplier == 1.0
        sp.cc = 1.0 * \
                (4 + alpha_cc * sp.mueff / N)**b / \
                (N**b + (4 + alpha_cc * 2 * sp.mueff / N)**b)
        sp.cc_sep = (1 + 1 / N + alpha_cc * sp.mueff / N) / (N**0.5 + 1 / N + alpha_cc * 2 * sp.mueff / N)  # \not\gg\cc
        if hasattr(opts['vv'], '__getitem__') and opts['vv'][0] == 'sweep_ccov1':
            ## meta_parameters.cc_multiplier == 1.0
            sp.cc = 1.0 * (4 + sp.mueff / N)**0.5 / ((N + 4)**0.5 + (2 * sp.mueff / N)**0.5)
        sp.rankmualpha = opts['CMA_rankmualpha']
        # sp.rankmualpha = _evalOption(opts['CMA_rankmualpha'], 0.3)
        ## meta_parameters.c1_multiplier == 1.0
        sp.c1 = ( 1.0 * ccovfac * min(1, sp.popsize / 6) *
                 ## meta_parameters.c1_exponent == 2.0
                 2 / ((N + 1.3)** 2.0 + sp.mueff))
        # 1/0
        sp.c1_sep = ccovfac * conedf(N, sp.mueff, N)
        if opts['CMA_rankmu'] != 0:  # also empty
            ## meta_parameters.cmu_multiplier == 2.0
            alphacov, mu = 2.0 , sp.mueff
            sp.cmu = min(1 - sp.c1, ccovfac * alphacov *
                         ## meta_parameters.cmu_exponent == 2.0
                         (sp.rankmualpha + mu - 2 + 1 / mu) / ((N + 2)** 2.0 + alphacov * mu / 2))
            if hasattr(opts['vv'], '__getitem__') and opts['vv'][0] == 'sweep_ccov':
                sp.cmu = opts['vv'][1]
            sp.cmu_sep = min(1 - sp.c1_sep, ccovfac * cmudf(N, sp.mueff, sp.rankmualpha))
        else:
            sp.cmu = sp.cmu_sep = 0
        if hasattr(opts['vv'], '__getitem__') and opts['vv'][0] == 'sweep_ccov1':
            sp.c1 = opts['vv'][1]

        sp.neg = _BlancClass()
        if opts['CMA_active'] and opts['CMA_on']:
            # in principle we have mu_opt = popsize/2 + lam_mirr/2,
            # which means in particular weights should only be negative for q > 0.5+mirr_frac/2
            if 1 < 3: # seems most natural: continuation of log(lambda/2) - log(k) qqqqqqqqqqqqqqqqqqqqqqqqqq
                sp.neg.mu_f = popsize // 2  # not sure anymore what this is good for
                sp.neg.weights = array([log(k) - log(popsize/2 + 1/2) for k in np.arange(np.ceil(popsize/2 + 1.1/2), popsize + .1)])
            sp.neg.mu = len(sp.neg.weights)
            sp.neg.weights /= sum(sp.neg.weights)
            sp.neg.mueff = 1 / sum(sp.neg.weights**2)
            ## meta_parameters.cact_exponent == 1.5
            sp.neg.cmuexp = opts['CMA_active'] * 0.3 * sp.neg.mueff / ((N + 2)** 1.5 + 1.0 * sp.neg.mueff)
            if hasattr(opts['vv'], '__getitem__') and opts['vv'][0] == 'sweep_ccov_neg':
                sp.neg.cmuexp = opts['vv'][1]
            # reasoning on learning rate cmuexp: with sum |w| == 1 and
            #   length-normalized vectors in the update, the residual
            #   variance in any direction exceeds exp(-N*cmuexp)
            assert sp.neg.mu >= sp.lam_mirr  # not really necessary
            # sp.neg.minresidualvariance = 0.66  # not it use, keep at least 0.66 in all directions, small popsize is most critical
        else:
            sp.neg.cmuexp = 0

        sp.CMA_on = sp.c1 + sp.cmu > 0
        # print(sp.c1_sep / sp.cc_sep)

        if not opts['CMA_on'] and opts['CMA_on'] not in (None, [], (), ''):
            sp.CMA_on = False
            # sp.c1 = sp.cmu = sp.c1_sep = sp.cmu_sep = 0
        mueff_exponent = 0.5
        if 1 < 3:
            mueff_exponent = opts['CSA_damp_mueff_exponent']
        # TODO: this will disappear, as it is done in class CMAAdaptSigmaCSA
        sp.damps = opts['CSA_dampfac'] * (0.5 +
                                          0.5 * min([1, (sp.lam_mirr / (0.159 * sp.popsize) - 1)**2])**1 +
                                          2 * max([0, ((sp.mueff - 1) / (N + 1))**mueff_exponent - 1]) + sp.cs
                                          )
        sp.cmean = float(opts['CMA_cmean'])
        # sp.kappa = 1  # 4-D, lam=16, rank1, kappa < 4 does not influence convergence rate
                        # in larger dim it does, 15-D with defaults, kappa=8 factor 2
        if verbose:
            if not sp.CMA_on:
                print('covariance matrix adaptation turned off')
            if opts['CMA_mu'] != None:
                print('mu = %f' % (sp.mu_f))
168926
def tensor(x:Any, *rest)->Tensor:
    ""
    if len(rest): x = (x,)+rest
    # XXX: Pytorch bug in dataloader using num_workers>0; TODO: create repro and report
    if is_listy(x) and len(x)==0: return tensor(0)
    res = torch.tensor(x) if is_listy(x) else as_tensor(x)
    if res.dtype is torch.int32:
        warn('Tensor is int32: upgrading to int64; for better performance use int64 input')
        return res.long()
    return res
597414
def write_cyc(fn, this, conv=1.0):
    

    lattice = this.get_cell()

    f = paropen(fn, "w")
    f.write("<------- Simulation box definition\n")
    f.write("<------- Barostat (on = 1, off = 0)\n")
    f.write("  0\n")
    f.write("<------- Box vectors (start)\n")
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][0]*conv, lattice[1][0]*conv, lattice[2][0]*conv))
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][1]*conv, lattice[1][1]*conv, lattice[2][1]*conv))
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][2]*conv, lattice[1][2]*conv, lattice[2][2]*conv))
    f.write("<------- Box vectors (end)\n")
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][0]*conv, lattice[1][0]*conv, lattice[2][0]*conv))
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][1]*conv, lattice[1][1]*conv, lattice[2][1]*conv))
    f.write("  %20.10f %20.10f %20.10f\n" % (lattice[0][2]*conv, lattice[1][2]*conv, lattice[2][2]*conv))
    f.write("<------- Mass and gamma of the box (used in connection with the barostat)\n")
    f.write("  240 0.005\n")
    f.write("<------- Stress tensor (start)\n")
    f.write("  0 0 0\n")
    f.write("  0 0 0\n")
    f.write("  0 0 0\n")
    f.write("<------- Stress tensor (end)\n")
    f.write("  0 0 0\n")
    f.write("  0 0 0\n")
    f.write("  0 0 0\n")
    f.close()
93237
def batchnorm_2d(nf:int, norm_type:NormType=NormType.Batch):
    ""
    bn = nn.BatchNorm2d(nf)
    with torch.no_grad():
        bn.bias.fill_(1e-3)
        bn.weight.fill_(0. if norm_type==NormType.BatchZero else 1.)
    return bn
496782
def batch_augment(x, func, device='/CPU:0'):
  
  with tf.device(device):
    return tf.map_fn(func, x)
885099
def lmx_base():
  """"""
  hparams = transformer.transformer_tpu()
  # sharing is counterproductive when underparameterized
  hparams.shared_embedding_and_softmax_weights = False
  # we judge by log-ppl, so label smoothing hurts.
  hparams.label_smoothing = 0.0
  # This makes the batch size on GPU the same as on TPU for a packed problem
  # with sequence length 256.
  # TODO(noam): fix the mess that is the data reading pipeline.
  hparams.max_length = 256
  # larger batch since we only have a decoder
  hparams.batch_size = 4096
  # save some memory so we can have a larger model
  hparams.activation_dtype = "bfloat16"
  return hparams
727701
def compute(self, config, budget, working_directory, *args, **kwargs):
		

		# device = torch.device('cpu')
		model = MNISTConvNet(num_conv_layers=config['num_conv_layers'],
							num_filters_1=config['num_filters_1'],
							num_filters_2=config['num_filters_2'] if 'num_filters_2' in config else None,
							num_filters_3=config['num_filters_3'] if 'num_filters_3' in config else None,
							dropout_rate=config['dropout_rate'],
							num_fc_units=config['num_fc_units'],
							kernel_size=3
		)

		criterion = torch.nn.CrossEntropyLoss()
		if config['optimizer'] == 'Adam':
			optimizer = torch.optim.Adam(model.parameters(), lr=config['lr'])
		else:
			optimizer = torch.optim.SGD(model.parameters(), lr=config['lr'], momentum=config['sgd_momentum'])

		for epoch in range(int(budget)):
			loss = 0
			model.train()
			for i, (x, y) in enumerate(self.train_loader):
				optimizer.zero_grad()
				output = model(x)
				loss = F.nll_loss(output, y)
				loss.backward()
				optimizer.step()

		train_accuracy = self.evaluate_accuracy(model, self.train_loader)
		validation_accuracy = self.evaluate_accuracy(model, self.validation_loader)
		test_accuracy = self.evaluate_accuracy(model, self.test_loader)

		return ({
			'loss': 1-validation_accuracy, # remember: HpBandSter always minimizes!
			'info': {	'test accuracy': test_accuracy,
						'train accuracy': train_accuracy,
						'validation accuracy': validation_accuracy,
						'number of parameters': model.number_of_parameters(),
					}
						
		})
774009
def fc(self, output_nodes, keep_prob=1, activation_fn=tf.nn.relu, b_value=0.0, s_value=1.0, bn=True,
           trainable=True):
        
        self.count['fc'] += 1
        scope = 'fc_' + str(self.count['fc'])
        with tf.variable_scope(scope):

            # Flatten if necessary
            if len(self.input.get_shape()) == 4:
                input_nodes = tf.Dimension(
                    self.input.get_shape()[1] * self.input.get_shape()[2] * self.input.get_shape()[3])
                output_shape = tf.stack([-1, input_nodes])
                self.input = tf.reshape(self.input, output_shape)

            # Matrix Multiplication Function
            input_nodes = self.input.get_shape()[1]
            output_shape = [input_nodes, output_nodes]
            w = self.weight_variable(name='weights', shape=output_shape, trainable=trainable)
            self.input = tf.matmul(self.input, w)

            if bn is True:  # batch normalization
                self.input = self.batch_norm(self.input, 'fc')
            if b_value is not None:  # bias value
                b = self.const_variable(name='bias', shape=[output_nodes], value=b_value, trainable=trainable)
                self.input = tf.add(self.input, b)
            if s_value is not None:  # scale value
                s = self.const_variable(name='scale', shape=[output_nodes], value=s_value, trainable=trainable)
                self.input = tf.multiply(self.input, s)
            if activation_fn is not None:  # activation function
                self.input = activation_fn(self.input)
            if keep_prob != 1:  # dropout function
                self.input = tf.nn.dropout(self.input, keep_prob=keep_prob)
        print(scope + ' output: ' + str(self.input.get_shape()))
1294795
def begin_update(self, x_data, h_0=None, drop=0.0):
        
        x_var = torch.autograd.Variable(xp2torch(x_data), requires_grad=True)

        # Make prediction
        out, h_n = self._model(x_var, h_0)
        # Shapes will be:
        # out = seq_len, batch, hidden_size * num_directions
        # h_n = num_layers * num_directions, batch, hidden_size

        def backward_pytorch_rnn(d_data, sgd=None):
            dy_data, _ = d_data
            dout = xp2torch(dy_data)
            torch.autograd.backward((out,), grad_tensors=(dout,))
            if sgd is not None:
                if self._optimizer is None:
                    self._optimizer = self._create_optimizer(sgd)
                self._optimizer.step()
                self._optimizer.zero_grad()
            return torch2xp(x_var.grad)

        return (torch2xp(out), h_n), backward_pytorch_rnn
434535
def generate_np(self, x_val, **kwargs):
    
    tfe = tf.contrib.eager
    x = tfe.Variable(x_val)
    adv_x = self.generate(x, **kwargs)
    return adv_x.numpy()
1304170
def bn2float(module:nn.Module)->nn.Module:
    ""
    if isinstance(module, torch.nn.modules.batchnorm._BatchNorm): module.float()
    for child in module.children(): bn2float(child)
    return module
1335475
def fit_zyz(target_gate):
    
    assert bk.BACKEND == 'eager'

    tf = bk.TL
    tfe = bk.tfe

    steps = 4000

    dev = '/gpu:0' if bk.DEVICE == 'gpu' else '/cpu:0'

    with tf.device(dev):
        t = tfe.Variable(np.random.normal(size=[3]), name='t')

        def loss_fn():
            """Loss"""
            gate = qf.ZYZ(t[0], t[1], t[2])
            ang = qf.fubini_study_angle(target_gate.vec, gate.vec)
            return ang

        loss_and_grads = tfe.implicit_value_and_gradients(loss_fn)
        # opt = tf.train.GradientDescentOptimizer(learning_rate=0.005)
        opt = tf.train.AdamOptimizer(learning_rate=0.001)
        # train = opt.minimize(ang, var_list=[t])

        for step in range(steps):
            loss, grads_and_vars = loss_and_grads()
            sys.stdout.write('\r')
            sys.stdout.write("step: {:3d} loss: {:10.9f}".format(step,
                                                                 loss.numpy()))
            if loss < 0.0001:
                break
            opt.apply_gradients(grads_and_vars)

        print()
    return bk.evaluate(t)