Evaluate a model for a single recording.

    Parameters
    ----------
    target_folder : string
        Folder where the model is
    test_file : string
        The test file (.hdf5)
18769
def _evaluate_model_single_file(target_folder, test_file):
    
    logging.info("Create running model...")
    model_src = get_latest_model(target_folder, "model")
    model_file_pointer = tempfile.NamedTemporaryFile(delete=False)
    model_use = model_file_pointer.name
    model_file_pointer.close()
    logging.info("Adjusted model is in %s.", model_use)
    create_adjusted_model_for_percentages(model_src, model_use)

    # Run evaluation
    project_root = get_project_root()
    time_prefix = time.strftime("%Y-%m-%d-%H-%M")
    logging.info("Evaluate '%s' with '%s'...", model_src, test_file)
    logfilefolder = os.path.join(project_root, "logs/")
    if not os.path.exists(logfilefolder):
        os.makedirs(logfilefolder)
    logfile = os.path.join(project_root,
                           "logs/%s-error-evaluation.log" %
                           time_prefix)
    with open(logfile, "w") as log, open(model_use, "r") as modl_src_p:
        p = subprocess.Popen([get_nntoolkit(), 'run',
                              '--batch-size', '1',
                              '-f%0.4f', test_file],
                             stdin=modl_src_p,
                             stdout=log)
        ret = p.wait()
        if ret != 0:
            logging.error("nntoolkit finished with ret code %s",
                          str(ret))
            sys.exit()
    return (logfile, model_use)
504501
def local_batch_predict(training_dir, prediction_input_file, output_dir, mode, batch_size,
                        shard_files, output_format):
  """"""
  # from . import predict as predict_module
  from .prediction import predict as predict_module

  if mode == 'evaluation':
    model_dir = os.path.join(training_dir, 'evaluation_model')
  elif mode == 'prediction':
    model_dir = os.path.join(training_dir, 'model')
  else:
    raise ValueError('mode must be evaluation or prediction')

  if not file_io.file_exists(model_dir):
    raise ValueError('Model folder %s does not exist' % model_dir)

  cmd = ['predict.py',
         '--predict-data=%s' % prediction_input_file,
         '--trained-model-dir=%s' % model_dir,
         '--output-dir=%s' % output_dir,
         '--output-format=%s' % output_format,
         '--batch-size=%s' % str(batch_size),
         '--shard-files' if shard_files else '--no-shard-files',
         '--has-target' if mode == 'evaluation' else '--no-has-target'
         ]

  # return predict_module.predict.main(cmd)
  return predict_module.main(cmd)
808867
def validate_metadata(target_dir, metadata):
    
    if not os.path.isdir(target_dir):
        print("Error: " + target_dir + " is not a directory")
        return False
    file_list = os.listdir(target_dir)
    for filename in file_list:
        if filename not in metadata:
            print("Error: " + filename + " present at" + target_dir +
                  " not found in metadata file")
            return False
    for filename in metadata:
        if filename not in file_list:
            print("Error: " + filename + " present in metadata file " +
                  " not found on disk at: " + target_dir)
            return False
    return True
282286
def prepare_files(self, target_dir):
        
        diff_names = self.git_wrapper.get_diff_names(self.remote_sha1, self.local_sha1)
        files_modified = diff_names.split('\n')
        extensions = LINTERS.keys()

        for file_path in files_modified:
            extension = file_path.split('.')[-1]
            if extension not in extensions:
                continue

            new_file_path = os.path.join(target_dir, file_path)
            new_dirname = os.path.dirname(new_file_path)
            if not os.path.isdir(new_dirname):
                os.makedirs(new_dirname)

            with open(new_file_path, "wb") as fh:
                self.git_wrapper.save_content_to_file(file_path, self.local_ref, fh)
            yield new_file_path
764577
def archive(self, target_path=None, zip_path=None):
        
        if target_path:
            self.target_path = target_path

        if zip_path:
            self.zip_path = zip_path

        if self.has_path is False or os.path.isdir(self.target_path) is False:
            raise RuntimeError("")

        zip = zipfile.ZipFile(
            self.zip_path,
            'w',
            zipfile.ZIP_DEFLATED
        )

        for root, _, files in os.walk(self.target_path):
            for file in files:
                if file in ARCHIVE_IGNORE_FILES:
                    continue

                current_dir = os.path.relpath(root, self.target_path)

                if current_dir == ".":
                    file_path = file
                else:
                    file_path = os.path.join(current_dir, file)

                print("Archive {}".format(file))

                zip.write(
                    os.path.join(root, file),
                    file_path
                )

        zip.close()
520136
def deploy_file(self,
                    file_name,
                    calc_md5=True,
                    calc_sha1=True,
                    parameters={}):
        
        if calc_md5:
            md5 = md5sum(file_name)
        if calc_sha1:
            sha1 = sha1sum(file_name)

        target = self

        if self.is_dir():
            target = self / pathlib.Path(file_name).name

        with open(file_name, 'rb') as fobj:
            target.deploy(fobj, md5, sha1, parameters)
1029666
def eval(self, restore_model_path: Optional[str]=None) -> None:
        

        saver = tf.train.Saver()
        with tf.Session(config=allow_growth_config) as sess:
            if restore_model_path:
                logger.info("restoring model from %s", restore_model_path)
                saver.restore(sess, restore_model_path)
            else:
                assert self.saved_model_path, "{}".format(self.saved_model_path)
                logger.info("restoring model from %s", self.saved_model_path)
                saver.restore(sess, self.saved_model_path)

            test_x, test_x_lens, test_y = self.corpus_reader.test_batch()

            feed_dict = {self.batch_x: test_x,
                         self.batch_x_lens: test_x_lens,
                         self.batch_y: test_y}

            test_ler, dense_decoded, dense_ref = sess.run(
                [self.ler, self.dense_decoded, self.dense_ref],
                feed_dict=feed_dict)
            hyps, refs = self.corpus_reader.human_readable_hyp_ref(
                dense_decoded, dense_ref)
            # Log hypotheses
            hyps_dir = os.path.join(self.exp_dir, "test")
            if not os.path.isdir(hyps_dir):
                os.mkdir(hyps_dir)
            with open(os.path.join(hyps_dir, "hyps"), "w",
                      encoding=ENCODING) as hyps_f:
                for hyp in hyps:
                    print(" ".join(hyp), file=hyps_f)
            with open(os.path.join(hyps_dir, "refs"), "w",
                      encoding=ENCODING) as refs_f:
                for ref in refs:
                    print(" ".join(ref), file=refs_f)

            test_per = utils.batch_per(hyps, refs)
            assert test_per == test_ler
            with open(os.path.join(hyps_dir, "test_per"), "w",
                      encoding=ENCODING) as per_f:
                print("LER: %f" % (test_ler), file=per_f)
849259
def starfit(folder, multiplicities=['single'], models='mist',
            use_emcee=False, plot_only=False, overwrite=False, verbose=False,
            logger=None, starmodel_type=None, ini_file='star.ini',
            no_plots=False, bands=None, **kwargs):
    
    nstars = {'single':1,
              'binary':2,
              'triple':3}

    if starmodel_type is None:
        Mod = StarModel
    else:
        Mod = starmodel_type

    ichrone = None

    print('Fitting {}'.format(folder))
    for mult in multiplicities:
        print('{} starfit...'.format(mult))
        model_filename = '{}_starmodel_{}.h5'.format(models, mult)

        #initialize logger for folder
        logfile = os.path.join(folder, 'starfit.log')
        logger = initLogging(logfile, logger)

        name = os.path.basename(os.path.abspath(folder))
        mod = None
        try:
            start = time.time()
            if plot_only:
                try:
                    mod = Mod.load_hdf('{}/{}'.format(folder,model_filename),
                                           name=name)
                except:
                    pass
            else:
                # Only try to fit model if it doesn't exist, unless overwrite is set
                fit_model = True

                try:
                    mod = Mod.load_hdf('{}/{}'.format(folder,model_filename),
                                         name=name)
                    fit_model = False
                except:
                    filename = '{}/{}'.format(folder,model_filename)
                    if os.path.exists(filename):
                        os.remove(filename)

                if fit_model or overwrite:
                    ini_file_path = os.path.join(folder, ini_file)
                    c = ConfigObj(ini_file_path)

                    if ichrone is None:
                        if bands is None:
                            bands = StarModel.get_bands(ini_file_path)
                        else:
                            bands += StarModel.get_bands(ini_file_path)
                        ichrone = get_ichrone(models, bands)

                    N = nstars[mult] if 'N' not in c else None
                    mod = Mod.from_ini(ichrone, folder, use_emcee=use_emcee, N=N,
                                        ini_file=ini_file, name=name)
                    try:
                        mod.obs.print_ascii()
                    except:
                        pass

                    # # Prime the jit call signature?
                    # ichrone.mag['J'](1.0, 9.7, 0.1, 1000, 0.4)

                    mod.fit(verbose=verbose, overwrite=overwrite, **kwargs)
                    mod.save_hdf(os.path.join(folder, model_filename), overwrite=overwrite)
                else:
                    logger.info('{} exists.  Use -o to overwrite.'.format(model_filename))

            # Only make corner plots if they are older
            #  than the starmodel hdf file
            make_corners = False
            for x in ['physical', 'observed']:
                f = os.path.join(folder,
                                 '{}_corner_{}_{}.png'.format(models, mult, x))
                if not os.path.exists(f):
                    make_corners = True
                    break
                else:
                    t_mod = os.path.getmtime(os.path.join(folder,model_filename))
                    t_plot = os.path.getmtime(f)
                    if t_mod > t_plot:
                        make_corners=True

            if (make_corners or plot_only) and not no_plots:
                corner_base = os.path.join(folder, '{}_corner_{}'.format(models, mult))
                fig1,fig2 = mod.corner_plots(corner_base)


            # Make mag plot if necessary.
            magplot_file = os.path.join(folder, '{}_mags_{}.png'.format(models, mult))
            make_magplot = False #True
            if os.path.exists(magplot_file):
                if os.path.getmtime(os.path.join(folder, model_filename)) > \
                        os.path.getmtime(magplot_file) or \
                        plot_only:
                    pass
                else:
                    make_magplot = False

            if make_magplot:
                fig = mod.mag_plot()
                plt.savefig(os.path.join(folder,'{}_mags_{}.png'.format(models, mult)))

            end = time.time()
            if plot_only:
                logger.info('{} starfit successful (plots only) for '.format(mult) +
                            '{} in {:.1f} minutes.'.format(folder, (end-start)/60))
            else:
                logger.info('{} starfit successful for '.format(mult) +
                            '{} in {:.1f} minutes.'.format(folder, (end-start)/60))
        except KeyboardInterrupt:
            logger.error('{} starfit calculation interrupted for {}.'.format(mult,folder))
            raise
        except:
            logger.error('{} starfit calculation failed for {}.'.format(mult,folder),
                         exc_info=True)

    return mod, logger
509723
def _get_original_targets(data):
    
    work_dir = os.path.join(_sv_workdir(data), "raw")
    batch = dd.get_batch(data) or dd.get_sample_name(data)
    return (glob.glob(os.path.join(work_dir, "*-%s.target.bed" % batch))[0],
            glob.glob(os.path.join(work_dir, "*-%s.antitarget.bed" % batch))[0])
496133
def targets(self):
        
        logging.info('Performing analysis with {} targets folder'.format(self.analysistype))
        for sample in self.runmetadata:
            if sample.general.bestassemblyfile != 'NA':
                sample[self.analysistype].targetpath = \
                    os.path.join(self.targetpath, 'genera', sample[self.analysistype].genus, '')
                # There is a relatively strict databasing scheme necessary for the custom targets. Eventually,
                # there will be a helper script to combine individual files into a properly formatted combined file
                try:
                    sample[self.analysistype].mappingfile = glob('{}*.fa'
                                                                 .format(sample[self.analysistype].targetpath))[0]
                # If the fasta file is missing, raise a custom error
                except IndexError as e:
                    # noinspection PyPropertyAccess
                    e.args = ['Cannot find the combined fasta file in {}. Please note that the file must have a '
                              '.fasta extension'.format(sample[self.analysistype].targetpath)]
                    if os.path.isdir(sample[self.analysistype].targetpath):
                        raise
                    else:
                        sample.general.bestassemblyfile = 'NA'
729650
def maybe_copy_file_to_directory(source_filepath, target_directory):
  
  if not tf.gfile.Exists(target_directory):
    tf.logging.info("Creating directory %s" % target_directory)
    os.mkdir(target_directory)
  target_filepath = os.path.join(target_directory,
                                 os.path.basename(source_filepath))
  if not tf.gfile.Exists(target_filepath):
    tf.logging.info("Copying %s to %s" % (source_filepath, target_filepath))
    tf.gfile.Copy(source_filepath, target_filepath)
    statinfo = os.stat(target_filepath)
    tf.logging.info("Successfully copied %s, %s bytes." % (target_filepath,
                                                           statinfo.st_size))
  else:
    tf.logging.info("Not copying, file already found: %s" % target_filepath)
  return target_filepath
226238
def train_model(model_folder):
    """"""
    os.chdir(model_folder)
    training = generate_training_command(model_folder)
    if training is None:
        return -1
    logging.info(training)
    os.chdir(model_folder)
    os.system(training)
496198
def update_if_outdated(folder):
    

    folders = []
    while os.path.isdir(folder):
        folders.append(folder)
        # Get info.yml
        with open(os.path.join(folder, "info.yml")) as ymlfile:
            content = yaml.load(ymlfile)
        folder = os.path.join(utils.get_project_root(), content['data-source'])
    raw_source_file = folder
    if not os.path.isfile(raw_source_file):
        logging.error("File '%s' was not found.", raw_source_file)
        logging.error("You should eventually execute 'hwrt download'.")
        sys.exit(-1)
    dt = os.path.getmtime(raw_source_file)
    source_mtime = datetime.datetime.utcfromtimestamp(dt)
    folders = folders[::-1]  # Reverse order to get the most "basic one first"

    for target_folder in folders:
        target_mtime = utils.get_latest_successful_run(target_folder)
        if target_mtime is None or source_mtime > target_mtime:
            # The source is later than the target. That means we need to
            # refresh the target
            if "preprocessed" in target_folder:
                logging.info("Preprocessed file was outdated. Update...")
                preprocess_dataset.main(os.path.join(utils.get_project_root(),
                                                     target_folder))
            elif "feature-files" in target_folder:
                logging.info("Feature file was outdated. Update...")
                create_ffiles.main(target_folder)
            elif "model" in target_folder:
                logging.info("Model file was outdated. Update...")
                create_model.main(target_folder, True)
            target_mtime = datetime.datetime.utcnow()
        else:
            logging.info("'%s' is up-to-date.", target_folder)
        source_mtime = target_mtime
848575
def model_segments(copy_file, work_dir, paired):
    
    out_file = os.path.join(work_dir, "%s.cr.seg" % dd.get_sample_name(paired.tumor_data))
    tumor_counts, normal_counts = heterogzygote_counts(paired)
    if not utils.file_exists(out_file):
        with file_transaction(paired.tumor_data, out_file) as tx_out_file:
            params = ["-T", "ModelSegments",
                      "--denoised-copy-ratios", copy_file,
                      "--allelic-counts", tumor_counts,
                      "--output-prefix", dd.get_sample_name(paired.tumor_data),
                      "-O", os.path.dirname(tx_out_file)]
            if normal_counts:
                params += ["--normal-allelic-counts", normal_counts]
            _run_with_memory_scaling(params, tx_out_file, paired.tumor_data)
            for tx_fname in glob.glob(os.path.join(os.path.dirname(tx_out_file),
                                                   "%s*" % dd.get_sample_name(paired.tumor_data))):
                shutil.copy(tx_fname, os.path.join(work_dir, os.path.basename(tx_fname)))
    return {"seg": out_file, "tumor_hets": out_file.replace(".cr.seg", ".hets.tsv"),
            "final_seg": out_file.replace(".cr.seg", ".modelFinal.seg")}
577238
private void eval(List<Tuple> training, List<Tuple> testing, int nfold) {
        classifier.train(training);

        for (Tuple tuple : testing) {
            String actual = classifier.predict(tuple).entrySet().stream()
                    .max((e1, e2) -> e1.getValue().compareTo(e2.getValue()))
                    .map(Map.Entry::getKey).orElse(StringUtils.EMPTY);
            updateScore(tuple, actual, nfold);
        }
    }
1006209
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Open file and parse into a data structure
        with open(path, 'r') as f:
            for line in f:
                sline = line.strip().split()
                if len(sline) == 1:
                    self.numParameters = sline[0]
                else:
                    # Create GSSHAPY TargetParameter object
                    target = TargetParameter(targetVariable=sline[0],
                                             varFormat=sline[1])

                    # Associate TargetParameter with ReplaceParamFile
                    target.replaceParamFile = self
1252421
def loadTargetsFromFile(self, filename, cols = None, everyNrows = 1,
                            delim = ' ', checkEven = 1):
        
        self.targets = self.loadVectors(filename, cols, everyNrows,
                                        delim, checkEven)