Writes training metrics to Tensorboard.
779398
def _write_metrics(self, iteration:int, last_metrics:MetricsList, start_idx:int=2)->None:
        ""
        recorder = self.learn.recorder
        for i, name in enumerate(recorder.names[start_idx:]):
            if last_metrics is None or len(last_metrics) < i+1: return
            scalar_value = last_metrics[i]
            self._write_scalar(name=name, scalar_value=scalar_value, iteration=iteration)
1172006
def _write_gen_model_stats(self, iteration:int)->None:
        ""
        generator = self.learn.gan_trainer.generator
        self.stats_writer.write(model=generator, iteration=iteration, tbwriter=self.tbwriter, name='gen_model_stats')
        self.gen_stats_updated = True
680597
def main():
    """"""
    print("|------- new changes!!!!!!!!!")
    # to get the dataset and net configuration
    train_data, val_data = get_dataset(dataset)
    netG = get_netG()
    netD = get_netD()
    loss, trainerG, trainerD = get_configurations(netG, netD)

    # set labels
    real_label = mx.nd.ones((opt.batch_size,), ctx=ctx)
    fake_label = mx.nd.zeros((opt.batch_size,), ctx=ctx)

    metric = mx.metric.Accuracy()
    print('Training... ')
    stamp = datetime.now().strftime('%Y_%m_%d-%H_%M')

    iter = 0

    # to metric the network
    loss_d = []
    loss_g = []
    inception_score = []

    for epoch in range(opt.nepoch):
        tic = time.time()
        btic = time.time()
        for data, _ in train_data:
            ############################
            # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
            ###########################
            # train with real_t
            data = data.as_in_context(ctx)
            noise = mx.nd.random.normal(0, 1, shape=(opt.batch_size, nz, 1, 1), ctx=ctx)

            with autograd.record():
                output = netD(data)
                # reshape output from (opt.batch_size, 2, 1, 1) to (opt.batch_size, 2)
                output = output.reshape((opt.batch_size, 2))
                errD_real = loss(output, real_label)

            metric.update([real_label, ], [output, ])

            with autograd.record():
                fake = netG(noise)
                output = netD(fake.detach())
                output = output.reshape((opt.batch_size, 2))
                errD_fake = loss(output, fake_label)
                errD = errD_real + errD_fake

            errD.backward()
            metric.update([fake_label,], [output,])

            trainerD.step(opt.batch_size)

            ############################
            # (2) Update G network: maximize log(D(G(z)))
            ###########################
            with autograd.record():
                output = netD(fake)
                output = output.reshape((-1, 2))
                errG = loss(output, real_label)

            errG.backward()

            trainerG.step(opt.batch_size)

            name, acc = metric.get()
            logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d'
                         , mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar(), acc, iter, epoch)
            if iter % niter == 0:
                visual('gout', fake.asnumpy(), name=os.path.join(outf, 'fake_img_iter_%d.png' % iter))
                visual('data', data.asnumpy(), name=os.path.join(outf, 'real_img_iter_%d.png' % iter))
                # record the metric data
                loss_d.append(errD)
                loss_g.append(errG)
                if opt.inception_score:
                    score, _ = get_inception_score(fake)
                    inception_score.append(score)

            iter = iter + 1
            btic = time.time()

        name, acc = metric.get()
        metric.reset()
        logging.info('\nbinary training acc at epoch %d: %s=%f', epoch, name, acc)
        logging.info('time: %f', time.time() - tic)

        # save check_point
        if check_point:
            netG.save_parameters(os.path.join(outf, 'generator_epoch_%d.params' %epoch))
            netD.save_parameters(os.path.join(outf, 'discriminator_epoch_%d.params' % epoch))

    # save parameter
    netG.save_parameters(os.path.join(outf, 'generator.params'))
    netD.save_parameters(os.path.join(outf, 'discriminator.params'))

    # visualization the inception_score as a picture
    if opt.inception_score:
        ins_save(inception_score)
839376
final void doScoring(Frame _train, Frame _trainOrig, Frame _valid, Frame _validOrig) {
    ModelMetrics mm = makeMetrics(_train, _trainOrig, true, "Metrics reported on training frame");
    _output._training_metrics = mm;
    _output._scored_train[_output._ntrees].fillFrom(mm);
    addModelMetrics(mm);
    // Optional validation part
    if (_valid!=null) {
      mm = makeMetrics(_valid, _validOrig, false, "Metrics reported on validation frame");
      _output._validation_metrics = mm;
      _output._scored_valid[_output._ntrees].fillFrom(mm);
      addModelMetrics(mm);
    }
  }
691115
def _eval_summary(self, context: MonitorContext, feed_dict: Optional[Dict]=None) -> None:
        

        if self._summary is None:
            raise RuntimeError('TensorBoard monitor task should set the Tensorflow.Summary object')

        if context.session is None:
            raise RuntimeError('To run a TensorBoard monitor task the TF session object'
                               ' must be provided when creating an instance of the Monitor')

        summary = context.session.run(self._summary, feed_dict=feed_dict)
        self._file_writer.add_summary(summary, context.global_step)
        if self._flush_immediately:
            self.flush()
427974
def training_loop(self):
    """"""
    if not self.restarting:
      self._write_counters(self._local_step_at_start, self._global_step)

    tf.logging.info(
        "Training %s up to %d, %d to go", self.model_mode,
        self.target_local_step, self.steps_to_go
    )

    yield

    self._write_counters(self.target_local_step, -1)
842840
def format_metric_values(self, metric_values):
    """"""

    # Early in training, metric_values may actually be None.
    loss_str = 'N/A'
    accuracy_str = 'N/A'
    try:
      loss_str = 'loss: %.3f' % metric_values[0]
      accuracy_str = 'accuracy: %.3f' % metric_values[1]
    except (TypeError, IndexError):
      pass

    return '%s, %s' % (loss_str, accuracy_str)
5629
def train():
    """"""
    trainer = gluon.Trainer(model.collect_params(), args.optimizer, {'learning_rate': args.lr})

    train_data_loader, val_data_loader, test_data_loader \
        = dataprocessor.make_dataloader(data_train, data_val, data_test, args)

    best_valid_bleu = 0.0
    for epoch_id in range(args.epochs):
        log_avg_loss = 0
        log_avg_gnorm = 0
        log_wc = 0
        log_start_time = time.time()
        for batch_id, (src_seq, tgt_seq, src_valid_length, tgt_valid_length)\
                in enumerate(train_data_loader):
            # logging.info(src_seq.context) Context suddenly becomes GPU.
            src_seq = src_seq.as_in_context(ctx)
            tgt_seq = tgt_seq.as_in_context(ctx)
            src_valid_length = src_valid_length.as_in_context(ctx)
            tgt_valid_length = tgt_valid_length.as_in_context(ctx)
            with mx.autograd.record():
                out, _ = model(src_seq, tgt_seq[:, :-1], src_valid_length, tgt_valid_length - 1)
                loss = loss_function(out, tgt_seq[:, 1:], tgt_valid_length - 1).mean()
                loss = loss * (tgt_seq.shape[1] - 1) / (tgt_valid_length - 1).mean()
                loss.backward()
            grads = [p.grad(ctx) for p in model.collect_params().values()]
            gnorm = gluon.utils.clip_global_norm(grads, args.clip)
            trainer.step(1)
            src_wc = src_valid_length.sum().asscalar()
            tgt_wc = (tgt_valid_length - 1).sum().asscalar()
            step_loss = loss.asscalar()
            log_avg_loss += step_loss
            log_avg_gnorm += gnorm
            log_wc += src_wc + tgt_wc
            if (batch_id + 1) % args.log_interval == 0:
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ppl={:.4f}, gnorm={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'
                             .format(epoch_id, batch_id + 1, len(train_data_loader),
                                     log_avg_loss / args.log_interval,
                                     np.exp(log_avg_loss / args.log_interval),
                                     log_avg_gnorm / args.log_interval,
                                     wps / 1000, log_wc / 1000))
                log_start_time = time.time()
                log_avg_loss = 0
                log_avg_gnorm = 0
                log_wc = 0
        valid_loss, valid_translation_out = evaluate(val_data_loader)
        valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
        logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                     .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
        test_loss, test_translation_out = evaluate(test_data_loader)
        test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
        dataprocessor.write_sentences(valid_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_valid_out.txt').format(epoch_id))
        dataprocessor.write_sentences(test_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_test_out.txt').format(epoch_id))
        if valid_bleu_score > best_valid_bleu:
            best_valid_bleu = valid_bleu_score
            save_path = os.path.join(args.save_dir, 'valid_best.params')
            logging.info('Save best parameters to {}'.format(save_path))
            model.save_parameters(save_path)
        if epoch_id + 1 >= (args.epochs * 2) // 3:
            new_lr = trainer.learning_rate * args.lr_update_factor
            logging.info('Learning rate change to {}'.format(new_lr))
            trainer.set_learning_rate(new_lr)
    if os.path.exists(os.path.join(args.save_dir, 'valid_best.params')):
        model.load_parameters(os.path.join(args.save_dir, 'valid_best.params'))
    valid_loss, valid_translation_out = evaluate(val_data_loader)
    valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
    logging.info('Best model valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                 .format(valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
    test_loss, test_translation_out = evaluate(test_data_loader)
    test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
    logging.info('Best model test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                 .format(test_loss, np.exp(test_loss), test_bleu_score * 100))
    dataprocessor.write_sentences(valid_translation_out,
                                  os.path.join(args.save_dir, 'best_valid_out.txt'))
    dataprocessor.write_sentences(test_translation_out,
                                  os.path.join(args.save_dir, 'best_test_out.txt'))
992043
def train_classifier(download=True, parameters=None, ngram_range=(1, 1)):
    """"""
    if download:
        download_wiki()

    path = os.path.join(l.TOPDIR, 'train.json')
    training_set = json.load(open(path))
    path = os.path.join(l.TOPDIR, 'wiki.json')
    wiki_set = json.load(open(path))

    target_names = list(set([i['unit'] for i in training_set + wiki_set]))
    train_data, train_target = [], []
    for example in training_set + wiki_set:
        train_data.append(clean_text(example['text']))
        train_target.append(target_names.index(example['unit']))

    tfidf_model = TfidfVectorizer(sublinear_tf=True,
                                  ngram_range=ngram_range,
                                  stop_words='english')

    matrix = tfidf_model.fit_transform(train_data)

    if parameters is None:
        parameters = {'loss': 'log', 'penalty': 'l2', 'n_iter': 50,
                      'alpha': 0.00001, 'fit_intercept': True}

    clf = SGDClassifier(**parameters).fit(matrix, train_target)
    obj = {'tfidf_model': tfidf_model,
           'clf': clf,
           'target_names': target_names}
    path = os.path.join(l.TOPDIR, 'clf.pickle')
    pickle.dump(obj, open(path, 'w'))
558780
def _output(cls, fluents: Sequence[FluentPair]) -> Sequence[tf.Tensor]:
        ''''''
        output = []
        for _, fluent in fluents:
            tensor = fluent.tensor
            if tensor.dtype != tf.float32:
                tensor = tf.cast(tensor, tf.float32)
            output.append(tensor)
        return tuple(output)
983739
def _write_images(self, name:str, images:[Tensor])->None:
        ""
        tag = self.ds_type.name + ' ' + name
        self.tbwriter.add_image(tag=tag, img_tensor=vutils.make_grid(images, normalize=True), global_step=self.iteration)
1102992
def write(self):
        
        # write the next file for all fields
        for field_name in self.field_names:
            filename = self.generate_tensor_filename(field_name, self._num_tensors-1)
            self._tensors[field_name].save(filename, compressed=True)

        # write the current metadata to file
        json.dump(self._metadata, open(self.metadata_filename, 'w'),
                  indent=JSON_INDENT,
                  sort_keys=True)

        # update
        self._has_unsaved_data = False
900193
def write(self, proto):
    
    proto.filterDim = self.filterDim
    proto.outputDim = self.outputDim
    proto.batchSize = self.batchSize

    lossHistoryProto = proto.init("losses", len(self.losses))
    i = 0
    for iteration, loss in self.losses.iteritems():
      iterationLossHistoryProto = lossHistoryProto[i]
      iterationLossHistoryProto.iteration = iteration
      iterationLossHistoryProto.loss = float(loss)
      i += 1

    proto.iteration = self._iteration

    proto.basis = list(
      self.basis.flatten().astype(type('float', (float,), {}))
    )

    # training parameters
    proto.learningRate = self.learningRate
    proto.decayCycle = self.decayCycle
    proto.learningRateDecay = self.learningRateDecay

    # LCA parameters
    proto.numLcaIterations = self.numLcaIterations
    proto.lcaLearningRate = self.lcaLearningRate
    proto.thresholdDecay = self.thresholdDecay
    proto.minThreshold = self.minThreshold
    proto.thresholdType = self.thresholdType

    # debugging
    proto.verbosity = self.verbosity
    proto.showEvery = self.showEvery
    proto.seed = self.seed
126185
def make_prediction_output_tensors(args, features, input_ops, model_fn_ops,
                                   keep_target):
  """"""
  target_name = feature_transforms.get_target_name(features)
  key_names = get_key_names(features)

  outputs = {}
  outputs.update({key_name: tf.squeeze(input_ops.features[key_name])
                  for key_name in key_names})

  if is_classification_model(args.model):

    # build maps from ints to the origional categorical strings.
    class_names = read_vocab(args, target_name)
    table = tf.contrib.lookup.index_to_string_table_from_tensor(
        mapping=class_names,
        default_value='UNKNOWN')

    # Get the label of the input target.
    if keep_target:
      input_target_label = table.lookup(input_ops.features[target_name])
      outputs[PG_TARGET] = tf.squeeze(input_target_label)

    # TODO(brandondutra): get the score of the target label too.
    probabilities = model_fn_ops.predictions['probabilities']

    # if top_n == 0, this means use all the classes. We will use class names as
    # probabilities labels.
    if args.top_n == 0:
      predicted_index = tf.argmax(probabilities, axis=1)
      predicted = table.lookup(predicted_index)
      outputs.update({PG_CLASSIFICATION_FIRST_LABEL: predicted})
      probabilities_list = tf.unstack(probabilities, axis=1)
      for class_name, p in zip(class_names, probabilities_list):
        outputs[class_name] = p
    else:
      top_n = args.top_n

      # get top k labels and their scores.
      (top_k_values, top_k_indices) = tf.nn.top_k(probabilities, k=top_n)
      top_k_labels = table.lookup(tf.to_int64(top_k_indices))

      # Write the top_k values using 2*top_n columns.
      num_digits = int(math.ceil(math.log(top_n, 10)))
      if num_digits == 0:
        num_digits = 1
      for i in range(0, top_n):
        # Pad i based on the size of k. So if k = 100, i = 23 -> i = '023'. This
        # makes sorting the columns easy.
        padded_i = str(i + 1).zfill(num_digits)

        if i == 0:
          label_alias = PG_CLASSIFICATION_FIRST_LABEL
        else:
          label_alias = PG_CLASSIFICATION_LABEL_TEMPLATE % padded_i

        label_tensor_name = (tf.squeeze(
            tf.slice(top_k_labels, [0, i], [tf.shape(top_k_labels)[0], 1])))

        if i == 0:
          score_alias = PG_CLASSIFICATION_FIRST_SCORE
        else:
          score_alias = PG_CLASSIFICATION_SCORE_TEMPLATE % padded_i

        score_tensor_name = (tf.squeeze(
            tf.slice(top_k_values,
                     [0, i],
                     [tf.shape(top_k_values)[0], 1])))

        outputs.update({label_alias: label_tensor_name,
                        score_alias: score_tensor_name})

  else:
    if keep_target:
      outputs[PG_TARGET] = tf.squeeze(input_ops.features[target_name])

    scores = model_fn_ops.predictions['scores']
    outputs[PG_REGRESSION_PREDICTED_TARGET] = tf.squeeze(scores)

  return outputs
981932
def train(num_training_iterations, report_interval,
          reduce_learning_rate_interval):
  """"""

  # Build the computation graph.
  graph_tensors, dataset_train = build_graph(
      lstm_depth=FLAGS.lstm_depth, batch_size=FLAGS.batch_size,
      num_embedding=FLAGS.num_embedding, num_hidden=FLAGS.num_hidden,
      truncation_length=FLAGS.truncation_length,
      sample_length=FLAGS.sample_length, max_grad_norm=FLAGS.max_grad_norm,
      initial_learning_rate=FLAGS.learning_rate,
      reduce_learning_rate_multiplier=FLAGS.reduce_learning_rate_multiplier,
      optimizer_epsilon=FLAGS.optimizer_epsilon)

  # Configure a checkpoint saver.
  saver_hook = _configure_saver(FLAGS.checkpoint_dir,
                                FLAGS.checkpoint_interval)

  # Train the network.
  with tf.train.SingularMonitoredSession(
      hooks=[saver_hook], checkpoint_dir=FLAGS.checkpoint_dir) as sess:

    start_iteration = sess.run(graph_tensors["global_step"])

    for train_iteration in range(start_iteration, num_training_iterations):
      if (train_iteration + 1) % report_interval == 0:
        train_loss_v, valid_loss_v, _ = sess.run(
            (graph_tensors["train_loss"],
             graph_tensors["valid_loss"],
             graph_tensors["train_step"]))

        train_generated_string_v = sess.run(
            graph_tensors["train_generated_string"])

        train_generated_string_human = dataset_train.to_human_readable(
            (train_generated_string_v, 0), indices=[0])

        tf.logging.info("%d: Training loss %f. Validation loss %f. Sample = %s",
                        train_iteration,
                        train_loss_v,
                        valid_loss_v,
                        train_generated_string_human)
      else:
        train_loss_v, _ = sess.run((graph_tensors["train_loss"],
                                    graph_tensors["train_step"]))
        tf.logging.info("%d: Training loss %f.", train_iteration, train_loss_v)

      if (train_iteration + 1) % reduce_learning_rate_interval == 0:
        sess.run(graph_tensors["reduce_learning_rate"])
        tf.logging.info("Reducing learning rate.")

    test_loss = sess.run(graph_tensors["test_loss"])
    tf.logging.info("Test loss %f", test_loss)
245976
def main(argv):
    """"""
    tf_records = argv[1:]
    logging.info("Training on %s records: %s to %s",
                 len(tf_records), tf_records[0], tf_records[-1])
    with utils.logged_timer("Training"):
        train(*tf_records)
    if FLAGS.export_path:
        dual_net.export_model(FLAGS.export_path)
    if FLAGS.freeze:
        if FLAGS.use_tpu:
            dual_net.freeze_graph_tpu(FLAGS.export_path)
        else:
            dual_net.freeze_graph(FLAGS.export_path)
567167
def _generate_metrics(bam_fname, config_file, ref_file,
                      bait_file, target_file):
    
    with open(config_file) as in_handle:
        config = yaml.safe_load(in_handle)
    broad_runner = broad.runner_from_config(config)
    bam_fname = os.path.abspath(bam_fname)
    path = os.path.dirname(bam_fname)
    out_dir = os.path.join(path, "metrics")
    utils.safe_makedir(out_dir)
    with utils.chdir(out_dir):
        with tx_tmpdir() as tmp_dir:
            cur_bam = os.path.basename(bam_fname)
            if not os.path.exists(cur_bam):
                os.symlink(bam_fname, cur_bam)
            gen_metrics = PicardMetrics(broad_runner, tmp_dir)
            gen_metrics.report(cur_bam, ref_file,
                               _bam_is_paired(bam_fname),
                               bait_file, target_file)
    return out_dir