Helper function to get training data
766595
def read_input_data(filename):
    """"""
    logging.info('Opening file %s for reading input', filename)
    input_file = open(filename, 'r')
    data = []
    labels = []
    for line in input_file:
        tokens = line.split(',', 1)
        labels.append(tokens[0].strip())
        data.append(tokens[1].strip())
    return labels, data
447011
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'examples' in _dict:
            args['examples'] = [
                TrainingExample._from_dict(x) for x in (_dict.get('examples'))
            ]
        return cls(**args)
409138
def train_input_fn(params):
  """"""
  file_pattern = os.path.join(getattr(params, "data_dir", ""), "*encoded-train*")
  return _read_and_batch_from_files(
      file_pattern, params.batch_size, params.max_length, params.num_cpu_cores,
      shuffle=True, repeat=params.repeat_dataset)
46176
def train(self, params):
    
    # self.net = buildNetwork(params['encoding_num'] * params['num_lags'],
    #                         params['num_cells'],
    #                         params['encoding_num'],
    #                         bias=True,
    #                         outputbias=True)

    ds = SupervisedDataSet(params['encoding_num'] * params['num_lags'],
                           params['encoding_num'])
    history = self.window(self.history, params['learning_window'])

    n = params['encoding_num']
    for i in xrange(params['num_lags'], len(history)):
      targets = numpy.zeros((1, n))
      targets[0, :] = self.encoder.encode(history[i])

      features = numpy.zeros((1, n * params['num_lags']))
      for lags in xrange(params['num_lags']):
        features[0, lags * n:(lags + 1) * n] = self.encoder.encode(
          history[i - (lags + 1)])
      ds.addSample(features, targets)

    trainer = BackpropTrainer(self.net,
                              dataset=ds,
                              verbose=params['verbosity'] > 0)

    if len(history) > 1:
      trainer.trainEpochs(params['num_epochs'])
1180998
def train(args):
    """"""
    vocab, row, col, counts = get_train_data(args)
    model = GloVe(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
                  dropout=args.dropout, x_max=args.x_max, alpha=args.alpha,
                  weight_initializer=mx.init.Uniform(scale=1 / args.emsize))
    context = get_context(args)
    model.initialize(ctx=context)
    if not args.no_hybridize:
        model.hybridize(static_alloc=not args.no_static_alloc)

    optimizer_kwargs = dict(learning_rate=args.lr, eps=args.adagrad_eps)
    params = list(model.collect_params().values())
    try:
        trainer = mx.gluon.Trainer(params, 'groupadagrad', optimizer_kwargs)
    except ValueError:
        logging.warning('MXNet <= v1.3 does not contain '
                        'GroupAdaGrad support. Falling back to AdaGrad')
        trainer = mx.gluon.Trainer(params, 'adagrad', optimizer_kwargs)

    index_dtype = 'int32'
    if counts.shape[0] >= np.iinfo(np.int32).max:
        index_dtype = 'int64'
        logging.info('Co-occurrence matrix is large. '
                     'Using int64 to represent sample indices.')
    indices = mx.nd.arange(counts.shape[0], dtype=index_dtype)
    for epoch in range(args.epochs):
        # Logging variables
        log_wc = 0
        log_start_time = time.time()
        log_avg_loss = 0

        mx.nd.shuffle(indices, indices)  # inplace shuffle
        bs = args.batch_size
        num_batches = indices.shape[0] // bs
        for i in range(num_batches):
            batch_indices = indices[bs * i:bs * (i + 1)]
            ctx = context[i % len(context)]
            batch_row = row[batch_indices].as_in_context(ctx)
            batch_col = col[batch_indices].as_in_context(ctx)
            batch_counts = counts[batch_indices].as_in_context(ctx)
            with mx.autograd.record():
                loss = model(batch_row, batch_col, batch_counts)
                loss.backward()

            if len(context) == 1 or (i + 1) % len(context) == 0:
                trainer.step(batch_size=1)

            # Logging
            log_wc += loss.shape[0]
            log_avg_loss += loss.mean().as_in_context(context[0])
            if (i + 1) % args.log_interval == 0:
                # Forces waiting for computation by computing loss value
                log_avg_loss = log_avg_loss.asscalar() / args.log_interval
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'.format(
                                 epoch, i + 1, num_batches, log_avg_loss,
                                 wps / 1000, log_wc / 1000))
                log_dict = dict(
                    global_step=epoch * len(indices) + i * args.batch_size,
                    epoch=epoch, batch=i + 1, loss=log_avg_loss,
                    wps=wps / 1000)
                log(args, log_dict)

                log_start_time = time.time()
                log_avg_loss = 0
                log_wc = 0

            if args.eval_interval and (i + 1) % args.eval_interval == 0:
                with print_time('mx.nd.waitall()'):
                    mx.nd.waitall()
                with print_time('evaluate'):
                    evaluate(args, model, vocab, i + num_batches * epoch)

    # Evaluate
    with print_time('mx.nd.waitall()'):
        mx.nd.waitall()
    with print_time('evaluate'):
        evaluate(args, model, vocab, num_batches * args.epochs,
                 eval_analogy=not args.no_eval_analogy)

    # Save params
    with print_time('save parameters'):
        model.save_parameters(os.path.join(args.logdir, 'glove.params'))
648600
function getData (allFiles = [], config) {
  const files = allFiles.reduce((memo, iteratee) => {
    const filePath = path.parse(iteratee)
    if (config.include.includes(filePath.ext) && !config.exclude.includes(filePath.base)) {
      const newFile = {
        file: iteratee.replace(config.content, ''),
        dir: filePath.dir.replace(config.content, ''),
        name: filePath.name,
        base: filePath.base,
        ext: filePath.ext,
      }
      if (filePath.ext === '.md') {
        const file = fs.readFileSync(iteratee, { encoding: 'utf8' })
        const frontMatter = fm(file)
        newFile.attr = frontMatter.attributes
        newFile.body = frontMatter.body
      } else if (filePath.ext === '.png') {
        const image = new PNG.load(iteratee)
        newFile.width = image.width
        newFile.height = image.height
      }
      memo.push(newFile)
    }
    return memo
  }, [])
  return files
}
1206957
def get_data_feed(self, train_mode=True, qname_in='input', qname_out='output', input_mapping=None):
    """"""
    return TFNode.DataFeed(self.mgr, train_mode, qname_in, qname_out, input_mapping)
1083739
def build_net(self, is_training):
        """"""
        cfg = self.cfg
        with tf.device('/cpu:0'):
            word_embed = tf.get_variable(
                name='word_embed', initializer=self.embed, dtype=tf.float32, trainable=False)
            char_embed = tf.get_variable(name='char_embed',
                                         shape=[cfg.char_vcb_size,
                                                cfg.char_embed_dim],
                                         dtype=tf.float32)

        # [query_length, batch_size]
        self.query_word = tf.placeholder(dtype=tf.int32,
                                         shape=[None, None],
                                         name='query_word')
        self.query_mask = tf.placeholder(dtype=tf.float32,
                                         shape=[None, None],
                                         name='query_mask')
        # [batch_size]
        self.query_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None], name='query_lengths')

        # [passage_length, batch_size]
        self.passage_word = tf.placeholder(
            dtype=tf.int32, shape=[None, None], name='passage_word')
        self.passage_mask = tf.placeholder(
            dtype=tf.float32, shape=[None, None], name='passage_mask')
        # [batch_size]
        self.passage_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None], name='passage_lengths')

        if is_training:
            self.answer_begin = tf.placeholder(
                dtype=tf.int32, shape=[None], name='answer_begin')
            self.answer_end = tf.placeholder(
                dtype=tf.int32, shape=[None], name='answer_end')

        self.query_char_ids = tf.placeholder(dtype=tf.int32,
                                             shape=[
                                                 self.cfg.max_char_length, None, None],
                                             name='query_char_ids')
        # sequence_length, batch_size
        self.query_char_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None, None], name='query_char_lengths')

        self.passage_char_ids = tf.placeholder(dtype=tf.int32,
                                               shape=[
                                                   self.cfg.max_char_length, None, None],
                                               name='passage_char_ids')
        # sequence_length, batch_size
        self.passage_char_lengths = tf.placeholder(dtype=tf.int32,
                                                   shape=[None, None],
                                                   name='passage_char_lengths')

        query_char_states = self.build_char_states(char_embed=char_embed,
                                                   is_training=is_training,
                                                   reuse=False,
                                                   char_ids=self.query_char_ids,
                                                   char_lengths=self.query_char_lengths)

        passage_char_states = self.build_char_states(char_embed=char_embed,
                                                     is_training=is_training,
                                                     reuse=True,
                                                     char_ids=self.passage_char_ids,
                                                     char_lengths=self.passage_char_lengths)

        with tf.variable_scope("encoding") as scope:
            query_states = tf.concat([tf.nn.embedding_lookup(
                word_embed, self.query_word), query_char_states], axis=2)
            scope.reuse_variables()
            passage_states = tf.concat([tf.nn.embedding_lookup(
                word_embed, self.passage_word), passage_char_states], axis=2)
        passage_states = tf.transpose(passage_states, perm=[1, 0, 2])
        query_states = tf.transpose(query_states, perm=[1, 0, 2])
        self.passage_states = passage_states
        self.query_states = query_states

        output, output2 = graph_to_network(passage_states, query_states,
                                           self.passage_lengths, self.query_lengths,
                                           self.graph, self.cfg.dropout,
                                           is_training, num_heads=cfg.num_heads,
                                           rnn_units=cfg.rnn_units)

        passage_att_mask = self.passage_mask
        batch_size_x = tf.shape(self.query_lengths)
        answer_h = tf.zeros(
            tf.concat([batch_size_x, tf.constant([cfg.ptr_dim], dtype=tf.int32)], axis=0))

        answer_context = tf.reduce_mean(output2, axis=1)

        query_init_w = tf.get_variable(
            'query_init_w', shape=[output2.get_shape().as_list()[-1], cfg.ptr_dim])
        self.query_init = query_init_w
        answer_context = tf.matmul(answer_context, query_init_w)

        output = tf.transpose(output, perm=[1, 0, 2])

        with tf.variable_scope('answer_ptr_layer'):
            ptr_att = DotAttention('ptr',
                                   hidden_dim=cfg.ptr_dim,
                                   is_vanilla=self.cfg.att_is_vanilla,
                                   is_identity_transform=self.cfg.att_is_id,
                                   need_padding=self.cfg.att_need_padding)
            answer_pre_compute = ptr_att.get_pre_compute(output)
            ptr_gru = XGRUCell(hidden_dim=cfg.ptr_dim)
            begin_prob, begin_logits = ptr_att.get_prob(output, answer_context, passage_att_mask,
                                                        answer_pre_compute, True)
            att_state = ptr_att.get_att(output, begin_prob)
            (_, answer_h) = ptr_gru.call(inputs=att_state, state=answer_h)
            answer_context = answer_h
            end_prob, end_logits = ptr_att.get_prob(output, answer_context,
                                                    passage_att_mask, answer_pre_compute,
                                                    True)

        self.begin_prob = tf.transpose(begin_prob, perm=[1, 0])
        self.end_prob = tf.transpose(end_prob, perm=[1, 0])
        begin_logits = tf.transpose(begin_logits, perm=[1, 0])
        end_logits = tf.transpose(end_logits, perm=[1, 0])

        if is_training:
            def label_smoothing(inputs, masks, epsilon=0.1):
                """Modify target for label smoothing."""
                epsilon = cfg.labelsmoothing
                num_of_channel = tf.shape(inputs)[-1]  # number of channels
                inputs = tf.cast(inputs, tf.float32)
                return (((1 - epsilon) * inputs) + (epsilon /
                                                    tf.cast(num_of_channel, tf.float32))) * masks
            cost1 = tf.reduce_mean(
                tf.losses.softmax_cross_entropy(label_smoothing(
                    tf.one_hot(self.answer_begin,
                               depth=tf.shape(self.passage_word)[0]),
                    tf.transpose(self.passage_mask, perm=[1, 0])), begin_logits))
            cost2 = tf.reduce_mean(
                tf.losses.softmax_cross_entropy(
                    label_smoothing(tf.one_hot(self.answer_end,
                                               depth=tf.shape(self.passage_word)[0]),
                                    tf.transpose(self.passage_mask, perm=[1, 0])), end_logits))

            reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
            l2_loss = tf.reduce_sum(reg_ws)
            loss = cost1 + cost2 + l2_loss
            self.loss = loss

            optimizer = tf.train.AdamOptimizer(learning_rate=cfg.learning_rate)
            self.train_op = optimizer.minimize(self.loss)

        return tf.stack([self.begin_prob, self.end_prob])
1154973
public function get_training_data() {

        \core_analytics\manager::check_can_manage_models();

        $timesplittingid = $this->get_time_splitting()->get_id();
        return \core_analytics\dataset_manager::export_training_data($this->get_id(), $timesplittingid);
    }
1241577
def get_data(self, request=None):
        """"""
        if request is None:
            raise ValueError
        data = [[] for _ in self.sources]
        for i in range(request):
            try:
                for source_data, example in zip(
                        data, next(self.child_epoch_iterator)):
                    source_data.append(example)
            except StopIteration:
                # If some data has been extracted and `strict` is not set,
                # we should spit out this data before stopping iteration.
                if not self.strictness and data[0]:
                    break
                elif self.strictness > 1 and data[0]:
                    raise ValueError
                raise
        return tuple(numpy.asarray(source_data) for source_data in data)
1341226
def train(data_train, model, nsp_loss, mlm_loss, vocab_size, ctx, store):
    """"""
    mlm_metric = nlp.metric.MaskedAccuracy()
    nsp_metric = nlp.metric.MaskedAccuracy()
    mlm_metric.reset()
    nsp_metric.reset()

    lr = args.lr
    optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
    if args.dtype == 'float16':
        optim_params['multi_precision'] = True

    trainer = mx.gluon.Trainer(model.collect_params(), 'bertadam', optim_params,
                               update_on_kvstore=False, kvstore=store)
    dynamic_loss_scale = args.dtype == 'float16'
    fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale)

    if args.ckpt_dir and args.start_step:
        state_path = os.path.join(args.ckpt_dir, '%07d.states' % args.start_step)
        logging.info('Loading trainer state from %s', state_path)
        trainer.load_states(state_path)

    accumulate = args.accumulate
    num_train_steps = args.num_steps
    warmup_ratio = args.warmup_ratio
    num_warmup_steps = int(num_train_steps * warmup_ratio)
    params = [p for p in model.collect_params().values() if p.grad_req != 'null']
    param_dict = model.collect_params()

    # Do not apply weight decay on LayerNorm and bias terms
    for _, v in model.collect_params('.*beta|.*gamma|.*bias').items():
        v.wd_mult = 0.0
    if accumulate > 1:
        for p in params:
            p.grad_req = 'add'

    train_begin_time = time.time()
    begin_time = time.time()
    running_mlm_loss = running_nsp_loss = running_num_tks = 0
    batch_num = 0
    step_num = args.start_step

    parallel_model = ParallelBERT(model, mlm_loss, nsp_loss, vocab_size,
                                  store.num_workers * accumulate, trainer=fp16_trainer)
    num_ctxes = len(ctx)
    parallel = nlp.utils.Parallel(num_ctxes if num_ctxes > 1 else 0, parallel_model)

    while step_num < num_train_steps:
        for _, dataloader in enumerate(data_train):
            if step_num >= num_train_steps:
                break

            # create dummy data loader if needed
            if args.dummy_data_len:
                target_shape = (args.batch_size*num_ctxes, args.dummy_data_len)
                dataloader = get_dummy_dataloader(dataloader, target_shape)

            for _, data_batch in enumerate(dataloader):
                if step_num >= num_train_steps:
                    break
                if batch_num % accumulate == 0:
                    step_num += 1
                    # if accumulate > 1, grad_req is set to 'add', and zero_grad is required
                    if accumulate > 1:
                        param_dict.zero_grad()
                    # update learning rate
                    if step_num <= num_warmup_steps:
                        new_lr = lr * step_num / num_warmup_steps
                    else:
                        offset = lr * step_num / num_train_steps
                        new_lr = lr - offset
                    trainer.set_learning_rate(new_lr)
                    if args.profile:
                        profile(step_num, 10, 12, profile_name=args.profile)
                if args.use_avg_len:
                    data_list = [[seq.as_in_context(context) for seq in shard]
                                 for context, shard in zip(ctx, data_batch)]
                else:
                    if data_batch[0].shape[0] < len(ctx):
                        continue
                    data_list = split_and_load(data_batch, ctx)

                ns_label_list, ns_pred_list = [], []
                mask_label_list, mask_pred_list, mask_weight_list = [], [], []

                # parallel forward / backward
                for data in data_list:
                    parallel.put(data)
                for _ in range(len(ctx)):
                    (_, next_sentence_label, classified, masked_id,
                     decoded, masked_weight, ls1, ls2, valid_length) = parallel.get()
                    ns_label_list.append(next_sentence_label)
                    ns_pred_list.append(classified)
                    mask_label_list.append(masked_id)
                    mask_pred_list.append(decoded)
                    mask_weight_list.append(masked_weight)
                    running_mlm_loss += ls1.as_in_context(mx.cpu()) / num_ctxes
                    running_nsp_loss += ls2.as_in_context(mx.cpu()) / num_ctxes
                    running_num_tks += valid_length.sum().as_in_context(mx.cpu())

                # update
                if (batch_num + 1) % accumulate == 0:
                    fp16_trainer.step(1, max_norm=1)
                nsp_metric.update(ns_label_list, ns_pred_list)
                mlm_metric.update(mask_label_list, mask_pred_list, mask_weight_list)
                # logging
                if (step_num + 1) % (args.log_interval) == 0 and (batch_num + 1) % accumulate == 0:
                    log(begin_time, running_num_tks, running_mlm_loss / accumulate,
                        running_nsp_loss / accumulate, step_num, mlm_metric,
                        nsp_metric, trainer, args.log_interval)
                    begin_time = time.time()
                    running_mlm_loss = running_nsp_loss = running_num_tks = 0
                    mlm_metric.reset_local()
                    nsp_metric.reset_local()

                # saving checkpoints
                if args.ckpt_dir and (step_num + 1) % (args.ckpt_interval) == 0 \
                   and (batch_num + 1) % accumulate == 0:
                    save_params(step_num, model, trainer, args.ckpt_dir)
                batch_num += 1
    save_params(step_num, model, trainer, args.ckpt_dir)
    mx.nd.waitall()
    train_end_time = time.time()
    logging.info('Train cost={:.1f}s'.format(train_end_time - train_begin_time))
94200
def from_learners(cls, learn_gen:Learner, learn_crit:Learner, switcher:Callback=None,
                      weights_gen:Tuple[float,float]=None, **learn_kwargs):
        ""
        losses = gan_loss_from_func(learn_gen.loss_func, learn_crit.loss_func, weights_gen=weights_gen)
        return cls(learn_gen.data, learn_gen.model, learn_crit.model, *losses, switcher=switcher, **learn_kwargs)
833788
def get_model(model_fn, train_data, param):
    
    model_param = merge_dicts({"train_data": train_data}, param["model"], param.get("shared", {}))
    return model_fn(**model_param)
1017841
def get_skill_data(self):
        
        path_to_sha = {
            folder: sha for folder, sha in self.get_shas()
        }
        modules = self.read_file('.gitmodules').split('[submodule "')
        for i, module in enumerate(modules):
            if not module:
                continue
            try:
                name = module.split('"]')[0].strip()
                path = module.split('path = ')[1].split('\n')[0].strip()
                url = module.split('url = ')[1].strip()
                sha = path_to_sha.get(path, '')
                yield name, path, url, sha
            except (ValueError, IndexError) as e:
                LOG.warning('Failed to parse submodule "{}" #{}:{}'.format(
                    locals().get('name', ''), i, e
                ))
1074374
def get_data(upd, bot_ident):
    """"""

    update_content = json.loads(upd.decode())
    result = update_content['result']
    data = (get_fields(update, bot_ident) for update in result)
    return (dt for dt in data if dt is not None)
264308
def train_batch_gen(self) -> Iterator:
        

        if len(self.train_fns) == 0:
            raise PersephoneException("""No training data available; cannot
                                       generate training batches.""")

        # Create batches of batch_size and shuffle them.
        fn_batches = self.make_batches(self.train_fns)


        if self.rand:
            random.shuffle(fn_batches)

        for fn_batch in fn_batches:
            logger.debug("Batch of training filenames: %s",
                          pprint.pformat(fn_batch))
            yield self.load_batch(fn_batch)
        else:
            raise StopIteration
868349
def get_actns(learn, hook:Hook, dl:DataLoader, pool=AdaptiveConcatPool2d, pool_dim:int=4, **kwargs):
        ""
        print('Getting activations...')

        actns = []
        learn.model.eval()
        with torch.no_grad():
            for (xb,yb) in progress_bar(dl):
                learn.model(xb)
                actns.append((hook.stored).cpu())

        if pool:
            pool = pool(pool_dim)
            return pool(torch.cat(actns)).view(len(dl.x),-1)
        else: return torch.cat(actns).view(len(dl.x),-1)