Store model output to laboratory archive.
506652
def archive(self):
        """"""

        # Traverse the model directory deleting symlinks, zero length files
        # and empty directories
        for path, dirs, files in os.walk(self.work_path, topdown=False):
            for f_name in files:
                f_path = os.path.join(path, f_name)
                if os.path.islink(f_path) or os.path.getsize(f_path) == 0:
                    os.remove(f_path)
            if len(os.listdir(path)) == 0:
                os.rmdir(path)
795951
def dump_to_store(dataset, store, writer=None, encoder=None,
                  encoding=None, unlimited_dims=None):
    """"""
    if writer is None:
        writer = ArrayWriter()

    if encoding is None:
        encoding = {}

    variables, attrs = conventions.encode_dataset_coordinates(dataset)

    check_encoding = set()
    for k, enc in encoding.items():
        # no need to shallow copy the variable again; that already happened
        # in encode_dataset_coordinates
        variables[k].encoding = enc
        check_encoding.add(k)

    if encoder:
        variables, attrs = encoder(variables, attrs)

    store.store(variables, attrs, check_encoding, writer,
                unlimited_dims=unlimited_dims)
1237364
def train_and_save(self, model_filename, cannon_data_filename, 
        clobber=False, **kwargs):
        

        if any(map(os.path.exists,
            (model_filename, cannon_data_filename))) and not clobber:
            raise IOError("output file already exists")

        trained = self.train_global(**kwargs)
        with open(cannon_data_filename, "wb") as fp:
            pickle.dump(trained, fp, -1)

        self._configuration["model_grid"]["cannon_data"] = cannon_data_filename
        logger.info("Cannon coefficients pickled to {}".format(
            cannon_data_filename))

        self.save(model_filename, clobber)

        return True
97801
private File saveExampleSet(ExampleSet outputSet, ZipModel model) throws IOException
	{
		logger.info("");

		//
		File tmp = File.createTempFile("train", null);
		tmp.deleteOnExit();
		//File tmp = new File("examples/train");

		BufferedWriter out = new BufferedWriter(new FileWriter(tmp));
		outputSet.write(out);
		out.close();

		// add the example set to the model
		model.add(tmp, "train");

		return tmp;
	}
1018316
def (self):
        """"""

        archive_data = {"packets": self.recv_msgs,
                        "dataset": self.dataset_name,
                        "num_packets": len(self.recv_msgs),
                        "created": rnow()}
        self.write_to_file(archive_data,
                           self.archive_file)
878630
def get_experiment_fn(args):
  

  def get_experiment(output_dir):
    # Merge schema, input features, and transforms.
    train_config = util.merge_metadata(args.preprocess_output_dir,
                                       args.transforms_file)

    # Get the model to train.
    estimator = util.get_estimator(output_dir, train_config, args)

    # Save a copy of the scehma and input to the model folder.
    schema_file = os.path.join(args.preprocess_output_dir, util.SCHEMA_FILE)

    # Make list of files to save with the trained model.
    additional_assets = {'features.json': args.transforms_file,
                         util.SCHEMA_FILE: schema_file}
    if util.is_classification_model(args.model_type):
      target_name = train_config['target_column']
      vocab_file_name = util.CATEGORICAL_ANALYSIS % target_name
      vocab_file_path = os.path.join(
          args.preprocess_output_dir, vocab_file_name)
      assert file_io.file_exists(vocab_file_path)
      additional_assets[vocab_file_name] = vocab_file_path

    export_strategy_target = util.make_export_strategy(
        train_config=train_config,
        args=args,
        keep_target=True,
        assets_extra=additional_assets)
    export_strategy_notarget = util.make_export_strategy(
        train_config=train_config,
        args=args,
        keep_target=False,
        assets_extra=additional_assets)

    input_reader_for_train = get_reader_input_fn(
        train_config=train_config,
        preprocess_output_dir=args.preprocess_output_dir,
        model_type=args.model_type,
        data_paths=args.train_data_paths,
        batch_size=args.train_batch_size,
        shuffle=True,
        num_epochs=args.num_epochs)

    input_reader_for_eval = get_reader_input_fn(
        train_config=train_config,
        preprocess_output_dir=args.preprocess_output_dir,
        model_type=args.model_type,
        data_paths=args.eval_data_paths,
        batch_size=args.eval_batch_size,
        shuffle=False,
        num_epochs=1)

    return tf.contrib.learn.Experiment(
        estimator=estimator,
        train_input_fn=input_reader_for_train,
        eval_input_fn=input_reader_for_eval,
        train_steps=args.max_steps,
        export_strategies=[export_strategy_target, export_strategy_notarget],
        min_eval_frequency=args.min_eval_frequency,
        eval_steps=None,
    )

  # Return a function to create an Experiment.
  return get_experiment
1373391
def zip_source_model(ssmLT, archive_zip='', log=logging.info):
    
    basedir = os.path.dirname(ssmLT)
    if os.path.basename(ssmLT) != 'ssmLT.xml':
        orig = ssmLT
        ssmLT = os.path.join(basedir, 'ssmLT.xml')
        with open(ssmLT, 'wb') as f:
            f.write(open(orig, 'rb').read())

    archive_zip = archive_zip or os.path.join(basedir, 'ssmLT.zip')
    if os.path.exists(archive_zip):
        sys.exit('%s exists already' % archive_zip)
    oq = mock.Mock(inputs={'source_model_logic_tree': ssmLT})
    checksum = readinput.get_checksum32(oq)
    checkfile = os.path.join(os.path.dirname(ssmLT), 'CHECKSUM.txt')
    with open(checkfile, 'w') as f:
        f.write(str(checksum))
    files = logictree.collect_info(ssmLT).smpaths + [
        os.path.abspath(ssmLT), os.path.abspath(checkfile)]
    general.zipfiles(files, archive_zip, log=log, cleanup=True)
    return archive_zip
739416
def _generate_examples(self, archive):
    """"""
    num_skipped = 0
    for fname, fobj in archive:
      res = _NAME_RE.match(fname)
      if not res:  # README file, ...
        continue
      label = res.group(1).lower()
      if tf.compat.as_bytes("JFIF") not in fobj.peek(10):
        num_skipped += 1
        continue
      yield {
          "image": fobj,
          "image/filename": fname,
          "label": label,
      }

    if num_skipped != _NUM_CORRUPT_IMAGES:
      raise ValueError("Expected %d corrupt images, but found %d" % (
          _NUM_CORRUPT_IMAGES, num_skipped))
    logging.warning("%d images were corrupted and were skipped", num_skipped)
1122438
def store_model(constrs, vars)
      # Store all constraints
      constrs.each do |constr|
        store_constraint constr
        @constraints << constr
      end

      # We store variables now since they didn't exist earlier
      vars.each_with_index do |var, i|
        var.index = i
        var.model = self
        store_variable var
      end
    end
1020629
def output_files(self):
        """"""
        for dep in self.subgraph.successors(self.address):
            dep_rule = self.subgraph.node[dep]['target_obj']
            for out_file in dep_rule.output_files:
                yield out_file
1002178
def save_data(self, idx):
        """"""
        for subseqs in self:
            if isinstance(subseqs, abctools.OutputSequencesABC):
                subseqs.save_data(idx)
226238
def train_model(model_folder):
    """"""
    os.chdir(model_folder)
    training = generate_training_command(model_folder)
    if training is None:
        return -1
    logging.info(training)
    os.chdir(model_folder)
    os.system(training)
1047189
function storeModel(model, permissionSchemaKey) {
    if (model && model.rolePermissions) {
        var obj = {
            permissionSchemaKey: permissionSchemaKey,
            rolePermissions: model.rolePermissions
        };
        exports.store(obj);
    }
}
1081981
def configure_pipeline(p, dataset, model_dir, output_csv, output_bq_table):
  """"""

  data = _util.get_sources_from_dataset(p, dataset, 'predict')
  if len(dataset.schema) == 2:
    output_schema = [
        {'name': 'image_url', 'type': 'STRING'},
        {'name': 'target', 'type': 'STRING'},
        {'name': 'predicted', 'type': 'STRING'},
        {'name': 'target_prob', 'type': 'FLOAT'},
        {'name': 'predicted_prob', 'type': 'FLOAT'},
    ]
  else:
    output_schema = [
        {'name': 'image_url', 'type': 'STRING'},
        {'name': 'predicted', 'type': 'STRING'},
        {'name': 'predicted_prob', 'type': 'FLOAT'},
    ]
  results = (data |
             'Load Images' >> beam.ParDo(LoadImagesDoFn()) |
             'Batch Inputs' >> beam.ParDo(EmitAsBatchDoFn(20)) |
             'Batch Predict' >> beam.ParDo(PredictBatchDoFn(model_dir)) |
             'Unbatch' >> beam.ParDo(UnbatchDoFn()) |
             'Process Results' >> beam.ParDo(ProcessResultsDoFn()))

  if output_csv is not None:
    schema_file = output_csv + '.schema.json'
    results_save = (results |
                    'Prepare For Output' >> beam.ParDo(MakeCsvLineDoFn()) |
                    'Write Csv Results' >> beam.io.textio.WriteToText(output_csv,
                                                                      shard_name_template=''))
    (results_save |
     'Sample One' >> beam.transforms.combiners.Sample.FixedSizeGlobally(1) |
     'Serialize Schema' >> beam.Map(lambda path: json.dumps(output_schema)) |
     'Write Schema' >> beam.io.textio.WriteToText(schema_file, shard_name_template=''))

  if output_bq_table is not None:
    # BigQuery sink takes schema in the form of 'field1:type1,field2:type2...'
    bq_schema_string = ','.join(x['name'] + ':' + x['type'] for x in output_schema)
    sink = beam.io.BigQuerySink(output_bq_table, schema=bq_schema_string,
                                write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)
    results | 'Write BQ Results' >> beam.io.Write(sink)
755749
def _collect_layer_outputs(mod, data, include_layer=None, max_num_examples=None, logger=None):
    """"""
    collector = _LayerOutputCollector(include_layer=include_layer, logger=logger)
    num_examples = _collect_layer_statistics(mod, data, collector, max_num_examples, logger)
    return collector.nd_dict, num_examples
684280
def save_data(self):
        """"""
        filename = os.path.join(
            self._get_prefix(),
            self.model.site
        )
        self.store.store_json(
            filename,
            self.model.get_dict()
        )
391327
def _collapse_to_cwl_record(samples, want_attrs, input_files):
    
    input_keys = sorted(list(set().union(*[d["cwl_keys"] for d in samples])), key=lambda x: (-len(x), tuple(x)))
    out = {}
    for key in input_keys:
        if key in want_attrs:
            key_parts = key.split("__")
            vals = []
            cur = []
            for d in samples:
                vals.append(_to_cwl(tz.get_in(key_parts, d), input_files))
                # Remove nested keys to avoid specifying multiple times
                cur.append(_dissoc_in(d, key_parts) if len(key_parts) > 1 else d)
            samples = cur
            out[key] = vals
    return out