Construct a host_call writing scalar summaries.

  Args:
    model_dir: String containing path to train

  Returns:
    (fn, args) Pair to be called by TPUEstimator as the host_call.
518345
def create_host_call(model_dir):
  
  graph = tf.get_default_graph()
  summaries = graph.get_collection(tf.GraphKeys.SUMMARIES)
  gs_t = tf.reshape(tf.to_int32(tf.train.get_global_step()), [1])
  summary_kwargs = collections.OrderedDict()
  for t in summaries:
    # TODO(aidangomez): enable ImageSummary support when we have a faster method
    # see @shibow's comment in cl/202344570
    if t.op.type not in ["ScalarSummary"]:
      tf.logging.warn("Ignoring unsupported tf.Summary type %s" % t.op.type)
      continue

    name = t.op.name
    tensor = t.op.inputs[1]
    if t.op.type == "ScalarSummary":
      assert tensor.shape.is_compatible_with([])
      if tensor.dtype == tf.int64:
        tensor = tf.to_int32(tensor)
      summary_kwargs["ScalarSummary" + name] = tf.reshape(tensor, [1])
    elif t.op.type == "ImageSummary":
      # TODO(aidangomez): as we move to support more types, update
      # common_layers.tpu_safe_image_summary
      if tensor.dtype != tf.float32:
        tf.logging.warn(
            "Currently T2T on TPU only supports ImageSummary of "
            "tf.float32-type Tensors. Skipping Tensor "
            "%s with dtype %s..." % (tensor.name, tensor.dtype))
        continue
      # tensor = tf.to_float(tensor)
      summary_kwargs["ImageSummary" + name] = tensor
  # When no supported summaries are found, don't create host_call. Otherwise,
  # TPU outfeed queue would enqueue global_step while host_call doesn't dequeue
  # it, eventually causing hang.
  if not summary_kwargs:
    return None
  summary_kwargs["global_step"] = gs_t
  log_info("summary_kwargs %s" % str(summary_kwargs))

  def host_call_fn(**kwargs):
    """Training host call. Creates summaries for training metrics.

    Args:
      **kwargs: Dict of {str: Tensor} , with `Tensor` of shape `[batch]`. Must
        contain key "global_step" with value of current global_step Tensor.

    Returns:
      List of summary ops to run on the CPU host.
    """
    gs = tf.to_int64(kwargs.pop("global_step")[0])
    with tf.contrib.summary.create_file_writer(model_dir).as_default():
      with tf.contrib.summary.always_record_summaries():
        # We need to use tf.contrib.summary in order to feed the `step`.
        for name, value in sorted(six.iteritems(kwargs)):
          if name.startswith("ScalarSummary"):
            name = name[len("ScalarSummary"):]
            tf.contrib.summary.scalar(
                name, tf.reduce_mean(tf.to_float(value)), step=gs)
          elif name.startswith("ImageSummary"):
            name = name[len("ImageSummary"):]
            tf.contrib.summary.image(name, value, step=gs)

        return tf.contrib.summary.all_summary_ops()

  return (host_call_fn, summary_kwargs)
628309
def run_toy_model_tpu():
  """"""
  tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
      FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)

  iterations_per_loop = FLAGS.iterations
  mesh_shape = mtf.convert_to_shape(FLAGS.mesh_shape)
  config = tpu_config.RunConfig(
      cluster=tpu_cluster_resolver,
      model_dir=FLAGS.model_dir,
      save_checkpoints_steps=None,  # Disable the default saver
      save_checkpoints_secs=None,  # Disable the default saver
      log_step_count_steps=iterations_per_loop,
      save_summary_steps=iterations_per_loop,
      tpu_config=tpu_config.TPUConfig(
          num_shards=mesh_shape.size,
          iterations_per_loop=iterations_per_loop,
          num_cores_per_replica=1,
          per_host_input_for_training=tpu_config.InputPipelineConfig.BROADCAST))
  classifier = tpu_estimator.TPUEstimator(
      use_tpu=True,
      model_fn=model_fn,
      config=config,
      train_batch_size=FLAGS.batch_size,
      eval_batch_size=FLAGS.batch_size)
  current_step = estimator_lib._load_global_step_from_checkpoint_dir(FLAGS.model_dir)  # pylint: disable=protected-access,line-too-long
  logging.info('Current step %d', current_step)
  if FLAGS.steps_per_checkpoint == 0:
    classifier.train(input_fn=ToyModelInput(), max_steps=FLAGS.train_steps)
    return
  while current_step < FLAGS.train_steps:
    next_checkpoint = min(current_step + FLAGS.steps_per_checkpoint,
                          FLAGS.train_steps)
    classifier.train(input_fn=ToyModelInput(), max_steps=next_checkpoint)
    current_step = next_checkpoint
    logging.info('Starting to evaluate.')
    eval_results = classifier.evaluate(
        input_fn=ToyModelInput(),
        steps=156)  # since we have 10000 examples and batch_size = 64 per host
    logging.info('Eval results: %s', eval_results)
246336
function buildModelUrl (protocol, host, apiRoot, modelName, id) {
  var result

  try {
    result = url.format({
      protocol: protocol,
      host: host,
      pathname: url.resolve('/', [apiRoot, modelName, id].join('/'))
    })
  } catch (e) {
    return ''
  }

  return result
}
858224
private static void summarizeGBMModel(ModelSummary summary, hex.gbm.GBM.GBMModel model) {
    // add generic fields such as column names
    summarizeModelCommonFields(summary, model);

    summary.model_algorithm = "GBM";

    JsonObject all_params = (model.get_params()).toJSON();
    summary.critical_parameters = whitelistJsonObject(all_params, GBM_critical_params);
    summary.secondary_parameters = whitelistJsonObject(all_params, GBM_secondary_params);
    summary.expert_parameters = whitelistJsonObject(all_params, GBM_expert_params);
  }
748966
def render(self, hosts, vars={}):
        
        if self.tpl_file.endswith(".tpl"):
            return self._render_mako(hosts, vars)
        elif self.tpl_file.endswith(".py"):
            return self._render_py(hosts, vars)
        else:
            raise ValueError("Don't know how to handle '{0}'".format(self.tpl_file))
1280524
def find_learning_rate_from_args(args: argparse.Namespace) -> None:
    
    params = Params.from_file(args.param_path, args.overrides)
    find_learning_rate_model(params, args.serialization_dir,
                             start_lr=args.start_lr,
                             end_lr=args.end_lr,
                             num_batches=args.num_batches,
                             linear_steps=args.linear,
                             stopping_factor=args.stopping_factor,
                             force=args.force)
507677
@Override
	public void loadModel(File directory)  throws IOException {
		stats = DeepModelIO.load(new File(directory,"YuvStatistics.txt"));

		SequenceAndParameters<Tensor_F32, Function<Tensor_F32>> sequence =
				new ParseBinaryTorch7().parseIntoBoof(new File(directory,"model.net"));

		network = sequence.createForward(3,inputSize,inputSize);
		tensorOutput = new Tensor_F32(WI(1,network.getOutputShape()));

		BorderType type = BorderType.valueOf(stats.border);
		localNorm = new ImageLocalNormalization<>(GrayF32.class, type);
		kernel = DataManipulationOps.create1D_F32(stats.kernel);
	}
376108
def main(model=None, new_model_name="animal", output_dir=None, n_iter=30):
    """"""
    random.seed(0)
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")
    # Add entity recognizer to model if it's not in the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pipe(ner)
    # otherwise, get it, so we can add labels to it
    else:
        ner = nlp.get_pipe("ner")

    ner.add_label(LABEL)  # add new entity label to entity recognizer
    # Adding extraneous labels shouldn't mess anything up
    ner.add_label("VEGETABLE")
    if model is None:
        optimizer = nlp.begin_training()
    else:
        optimizer = nlp.resume_training()
    move_names = list(ner.move_names)
    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
    with nlp.disable_pipes(*other_pipes):  # only train NER
        sizes = compounding(1.0, 4.0, 1.001)
        # batch up the examples using spaCy's minibatch
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            batches = minibatch(TRAIN_DATA, size=sizes)
            losses = {}
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(texts, annotations, sgd=optimizer, drop=0.35, losses=losses)
            print("Losses", losses)

    # test the trained model
    test_text = "Do you like horses?"
    doc = nlp(test_text)
    print("Entities in '%s'" % test_text)
    for ent in doc.ents:
        print(ent.label_, ent.text)

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.meta["name"] = new_model_name  # rename model
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        # Check the classes have loaded back consistently
        assert nlp2.get_pipe("ner").move_names == move_names
        doc2 = nlp2(test_text)
        for ent in doc2.ents:
            print(ent.label_, ent.text)
905995
def load_model(model_file):
    
    # Extract tar
    with tarfile.open(model_file) as tar:
        tarfolder = tempfile.mkdtemp()
        tar.extractall(path=tarfolder)

    from . import features
    from . import preprocessing

    # Get the preprocessing
    with open(os.path.join(tarfolder, "preprocessing.yml"), 'r') as ymlfile:
        preprocessing_description = yaml.load(ymlfile)
    preprocessing_queue = preprocessing.get_preprocessing_queue(
        preprocessing_description['queue'])

    # Get the features
    with open(os.path.join(tarfolder, "features.yml"), 'r') as ymlfile:
        feature_description = yaml.load(ymlfile)
    feature_str_list = feature_description['features']
    feature_list = features.get_features(feature_str_list)

    # Get the model
    import nntoolkit.utils
    model = nntoolkit.utils.get_model(model_file)

    output_semantics_file = os.path.join(tarfolder, 'output_semantics.csv')
    output_semantics = nntoolkit.utils.get_outputs(output_semantics_file)

    # Cleanup
    shutil.rmtree(tarfolder)

    return (preprocessing_queue, feature_list, model, output_semantics)
660782
def _create_events_writer(self, directory):
    
    total_size = 0
    events_files = self._fetch_events_files_on_disk()
    for file_name in events_files:
      file_path = os.path.join(self._events_directory, file_name)
      total_size += tf.io.gfile.stat(file_path).length

    if total_size >= self.total_file_size_cap_bytes:
      # The total size written to disk is too big. Delete events files until
      # the size is below the cap.
      for file_name in events_files:
        if total_size < self.total_file_size_cap_bytes:
          break

        file_path = os.path.join(self._events_directory, file_name)
        file_size = tf.io.gfile.stat(file_path).length
        try:
          tf.io.gfile.remove(file_path)
          total_size -= file_size
          logger.info(
              "Deleted %s because events files take up over %d bytes",
              file_path, self.total_file_size_cap_bytes)
        except IOError as err:
          logger.error("Deleting %s failed: %s", file_path, err)

    # We increment this index because each events writer must differ in prefix.
    self._events_file_count += 1
    file_path = "%s.%d.%d" % (
        os.path.join(directory, DEBUGGER_EVENTS_FILE_STARTING_TEXT),
        time.time(), self._events_file_count)
    logger.info("Creating events file %s", file_path)
    return pywrap_tensorflow.EventsWriter(tf.compat.as_bytes(file_path))
461125
def makedir(self):
        
        try:
            os.makedirs(self.model_dir)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise
659497
def export_serving(model_path):
    
    pred_config = PredictConfig(
        session_init=get_model_loader(model_path),
        model=InferenceOnlyModel(),
        input_names=['input_img_bytes'],
        output_names=['prediction_img_bytes'])
    ModelExporter(pred_config).export_serving('/tmp/exported')
1168471
def _default_output_dir():
  """"""
  try:
    dataset_name = gin.query_parameter("inputs.dataset_name")
  except ValueError:
    dataset_name = "random"
  dir_name = "{model_name}_{dataset_name}_{timestamp}".format(
      model_name=gin.query_parameter("train.model").configurable.name,
      dataset_name=dataset_name,
      timestamp=datetime.datetime.now().strftime("%Y%m%d_%H%M"),
  )
  dir_path = os.path.join("~", "trax", dir_name)
  print()
  trax.log("No --output_dir specified")
  return dir_path
1330142
def model(self,
              voltages=True,
              sensitivities=False,
              potentials=False,
              output_directory=None,
              silent=False,
              ):
        
        self._check_state()
        if self.can_model:
            if output_directory is not None:
                if not os.path.isdir(output_directory):
                    os.makedirs(output_directory)
                    tempdir = output_directory
                    self._model(voltages, sensitivities, potentials, tempdir)
                else:
                    raise IOError(
                        'output directory already exists: {0}'.format(
                            output_directory
                        )
                    )
            else:
                with tempfile.TemporaryDirectory(dir=self.tempdir) as tempdir:
                    self._model(
                        voltages, sensitivities, potentials, tempdir,
                        silent=silent
                    )

            return 1
        else:
            print('Sorry, not all required information to model are present')
            print('Check:')
            print('1) configurations present: self.configs.configs')
            print('2) is a model present')
            return None
1048124
def main(model_folder):
    """"""
    model_description_file = os.path.join(model_folder, "info.yml")

    # Read the model description file
    with open(model_description_file, 'r') as ymlfile:
        model_description = yaml.load(ymlfile)

    # Analyze model
    logging.info(model_description['model'])
    data = {}
    data['training'] = os.path.join(model_folder, "traindata.hdf5")
    data['testing'] = os.path.join(model_folder, "testdata.hdf5")
    data['validating'] = os.path.join(model_folder, "validdata.hdf5")
    train_model(model_folder)
1169753
def get_data_iters_and_vocabs(args: argparse.Namespace,
                              model_folder: Optional[str]) -> Tuple['data_io.BaseParallelSampleIter',
                                                                    List[vocab.Vocab], vocab.Vocab, model.ModelConfig]:
    

    model_config = model.SockeyeModel.load_config(os.path.join(args.model, C.CONFIG_NAME))

    if args.max_seq_len is None:
        max_seq_len_source = model_config.config_data.max_seq_len_source
        max_seq_len_target = model_config.config_data.max_seq_len_target
    else:
        max_seq_len_source, max_seq_len_target = args.max_seq_len

    batch_num_devices = 1 if args.use_cpu else sum(-di if di < 0 else 1 for di in args.device_ids)

    # Load the existing vocabs created when starting the training run.
    source_vocabs = vocab.load_source_vocabs(model_folder)
    target_vocab = vocab.load_target_vocab(model_folder)

    sources = [args.source] + args.source_factors
    sources = [str(os.path.abspath(source)) for source in sources]

    score_iter = data_io.get_scoring_data_iters(
        sources=sources,
        target=os.path.abspath(args.target),
        source_vocabs=source_vocabs,
        target_vocab=target_vocab,
        batch_size=args.batch_size,
        batch_num_devices=batch_num_devices,
        max_seq_len_source=max_seq_len_source,
        max_seq_len_target=max_seq_len_target)

    return score_iter, source_vocabs, target_vocab, model_config
1072011
def describe(self, model_name):
    
    model_yaml = yaml.safe_dump(self.get_model_details(model_name), default_flow_style=False)
    print(model_yaml)