setup observations from PEST-style SMP file pairs
1949
def setup_smp(self):
        
        if self.obssim_smp_pairs is None:
            return
        if len(self.obssim_smp_pairs) == 2:
            if isinstance(self.obssim_smp_pairs[0],str):
                self.obssim_smp_pairs = [self.obssim_smp_pairs]
        for obs_smp,sim_smp in self.obssim_smp_pairs:
            self.log("processing {0} and {1} smp files".format(obs_smp,sim_smp))
            if not os.path.exists(obs_smp):
                self.logger.lraise("couldn't find obs smp: {0}".format(obs_smp))
            if not os.path.exists(sim_smp):
                self.logger.lraise("couldn't find sim smp: {0}".format(sim_smp))
            new_obs_smp = os.path.join(self.m.model_ws,
                                              os.path.split(obs_smp)[-1])
            shutil.copy2(obs_smp,new_obs_smp)
            new_sim_smp = os.path.join(self.m.model_ws,
                                              os.path.split(sim_smp)[-1])
            shutil.copy2(sim_smp,new_sim_smp)
            pyemu.smp_utils.smp_to_ins(new_sim_smp)
284282
def iterifs(self):
        '''  '''
        if_path = os.path.join(SYSFS_NET_PATH, self.name, b"brif")
        net_files = os.listdir(if_path)
        for iface in net_files:
            yield iface
1248038
def sample_counters(mc, system_info):
    """"""
    return {
        (x, y): mc.get_router_diagnostics(x, y) for (x, y) in system_info
    }
987223
def init(conf):
    

    logger = logging.getLogger(__name__)

    # Initialize mimetypes library
    mimetypes.init([pkg_resources.resource_filename('ehforwarderbot', 'mimetypes')])

    # Initialize all channels
    # (Load libraries and modules and init them)

    for i in conf['slave_channels']:
        logger.log(99, "\x1b[0;37;46m %s \x1b[0m", _("Initializing slave {}...").format(i))

        cls = utils.locate_module(i, 'slave')
        telemetry_set_metadata({i: cls.__version__})
        instance_id = i.split('#', 1)[1:]
        instance_id = (instance_id and instance_id[0]) or None
        coordinator.add_channel(cls(instance_id=instance_id))

        logger.log(99, "\x1b[0;37;42m %s \x1b[0m",
                   _("Slave channel {name} ({id}) # {instance_id} is initialized.")
                   .format(name=cls.channel_name, id=cls.channel_id,
                           instance_id=instance_id or _("Default profile")))

    logger.log(99, "\x1b[0;37;46m %s \x1b[0m",
               _("Initializing master {}...").format(conf['master_channel']))
    instance_id = conf['master_channel'].split('#', 1)[1:]
    instance_id = (instance_id and instance_id[0]) or None
    module = utils.locate_module(conf['master_channel'], 'master')
    coordinator.add_channel(module(instance_id=instance_id))
    telemetry_set_metadata({conf['master_channel']: module.__version__})
    logger.log(99, "\x1b[0;37;42m %s \x1b[0m",
               _("Master channel {name} ({id}) # {instance_id} is initialized.")
               .format(name=coordinator.master.channel_name,
                       id=coordinator.master.channel_id,
                       instance_id=instance_id or _("Default profile")))

    logger.log(99, "\x1b[1;37;42m %s \x1b[0m", _("All channels initialized."))
    for i in conf['middlewares']:
        logger.log(99, "\x1b[0;37;46m %s \x1b[0m", _("Initializing middleware {}...").format(i))
        cls = utils.locate_module(i, 'middleware')
        telemetry_set_metadata({i: cls.__version__})

        instance_id = i.split('#', 1)[1:]
        instance_id = (instance_id and instance_id[0]) or None
        coordinator.add_middleware(cls(instance_id=instance_id))
        logger.log(99, "\x1b[0;37;42m %s \x1b[0m",
                   _("Middleware {name} ({id}) # {instance_id} is initialized.")
                   .format(name=cls.middleware_name, id=cls.middleware_id,
                           instance_id=instance_id or _("Default profile")))

    logger.log(99, "\x1b[1;37;42m %s \x1b[0m", _("All middlewares are initialized."))

    coordinator.master_thread = threading.Thread(target=coordinator.master.poll)
    coordinator.slave_threads = {key: threading.Thread(target=coordinator.slaves[key].poll)
                                 for key in coordinator.slaves}
1349368
def beacon(config):
    '''
    
    '''
    ret = []
    pkey = 'sh.vt'
    shells = _get_shells()
    ps_out = __salt__['status.procs']()
    track_pids = []
    for pid in ps_out:
        if any(ps_out[pid].get('cmd', '').lstrip('-') in shell for shell in shells):
            track_pids.append(pid)
    if pkey not in __context__:
        __context__[pkey] = {}
    for pid in track_pids:
        if pid not in __context__[pkey]:
            cmd = ['strace', '-f', '-e', 'execve', '-p', '{0}'.format(pid)]
            __context__[pkey][pid] = {}
            __context__[pkey][pid]['vt'] = salt.utils.vt.Terminal(
                    cmd,
                    log_stdout=True,
                    log_stderr=True,
                    stream_stdout=False,
                    stream_stderr=False)
            __context__[pkey][pid]['user'] = ps_out[pid].get('user')
    for pid in list(__context__[pkey]):
        out = ''
        err = ''
        while __context__[pkey][pid]['vt'].has_unread_data:
            tout, terr = __context__[pkey][pid]['vt'].recv()
            if not terr:
                break
            out += salt.utils.stringutils.to_unicode(tout or '')
            err += terr
        for line in err.split('\n'):
            event = {'args': [],
                     'tag': pid}
            if 'execve' in line:
                comps = line.split('execve')[1].split('"')
                for ind, field in enumerate(comps):
                    if ind == 1:
                        event['cmd'] = field
                        continue
                    if ind % 2 != 0:
                        event['args'].append(field)
                event['user'] = __context__[pkey][pid]['user']
                ret.append(event)
        if not __context__[pkey][pid]['vt'].isalive():
            __context__[pkey][pid]['vt'].close()
            __context__[pkey].pop(pid)
    return ret
1192504
def __setup_fileserver(self):
        '''
        
        '''
        # Avoid circular import
        import salt.fileserver
        self.fs_ = salt.fileserver.Fileserver(self.opts)
        self._serve_file = self.fs_.serve_file
        self._file_find = self.fs_._find_file
        self._file_hash = self.fs_.file_hash
        self._file_hash_and_stat = self.fs_.file_hash_and_stat
        self._file_list = self.fs_.file_list
        self._file_list_emptydirs = self.fs_.file_list_emptydirs
        self._dir_list = self.fs_.dir_list
        self._symlink_list = self.fs_.symlink_list
        self._file_envs = self.fs_.file_envs
1061873
def cli(ctx, board, serial_port, ftdi_id, sram, project_dir,
        verbose, verbose_yosys, verbose_arachne):
    """"""

    drivers = Drivers()
    drivers.pre_upload()
    # Run scons
    exit_code = SCons(project_dir).upload({
        'board': board,
        'verbose': {
            'all': verbose,
            'yosys': verbose_yosys,
            'arachne': verbose_arachne
        }
    }, serial_port, ftdi_id, sram)
    drivers.post_upload()
    ctx.exit(exit_code)
651403
def _prepare_smoove_bams(full_bams, sr_bams, disc_bams, items, tx_work_dir):
    
    input_dir = utils.safe_makedir(tx_work_dir)
    out = []
    for full, sr, disc, data in zip(full_bams, sr_bams, disc_bams, items):
        if sr and disc:
            new_full = os.path.join(input_dir, "%s.bam" % dd.get_sample_name(data))
            new_sr = os.path.join(input_dir, "%s.split.bam" % dd.get_sample_name(data))
            new_disc = os.path.join(input_dir, "%s.disc.bam" % dd.get_sample_name(data))
            utils.symlink_plus(full, new_full)
            utils.symlink_plus(sr, new_sr)
            utils.symlink_plus(disc, new_disc)
            out.append(new_full)
        else:
            out.append(full)
    return out
1081537
static List<ExperimentAlgorithm<BinarySolution, List<BinarySolution>>> configureAlgorithmList(
      List<ExperimentProblem<BinarySolution>> problemList) {
    List<ExperimentAlgorithm<BinarySolution, List<BinarySolution>>> algorithms = new ArrayList<>();
    for (int run = 0; run < INDEPENDENT_RUNS; run++) {

      for (int i = 0; i < problemList.size(); i++) {
        Algorithm<List<BinarySolution>> algorithm = new NSGAIIBuilder<BinarySolution>(
            problemList.get(i).getProblem(),
            new SinglePointCrossover(1.0),
            new BitFlipMutation(
                1.0 / ((BinaryProblem) problemList.get(i).getProblem()).getNumberOfBits(0)),
                100)
            .setMaxEvaluations(25000)
            .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }

      for (int i = 0; i < problemList.size(); i++) {
        Algorithm<List<BinarySolution>> algorithm = new SPEA2Builder<BinarySolution>(
            problemList.get(i).getProblem(),
            new SinglePointCrossover(1.0),
            new BitFlipMutation(
                1.0 / ((BinaryProblem) problemList.get(i).getProblem()).getNumberOfBits(0)))
            .setMaxIterations(250)
            .setPopulationSize(100)
            .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }

      for (int i = 0; i < problemList.size(); i++) {
        Algorithm<List<BinarySolution>> algorithm = new MOCellBuilder<BinarySolution>(
            problemList.get(i).getProblem(),
            new SinglePointCrossover(1.0),
            new BitFlipMutation(
                1.0 / ((BinaryProblem) problemList.get(i).getProblem()).getNumberOfBits(0)))
            .setMaxEvaluations(25000)
            .setPopulationSize(100)
            .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }

      for (int i = 0; i < problemList.size(); i++) {
        CrossoverOperator<BinarySolution> crossoverOperator;
        MutationOperator<BinarySolution> mutationOperator;
        SelectionOperator<List<BinarySolution>, BinarySolution> parentsSelection;
        SelectionOperator<List<BinarySolution>, List<BinarySolution>> newGenerationSelection;

        crossoverOperator = new HUXCrossover(1.0);
        parentsSelection = new RandomSelection<BinarySolution>();
        newGenerationSelection = new RankingAndCrowdingSelection<BinarySolution>(100);
        mutationOperator = new BitFlipMutation(0.35);
        Algorithm<List<BinarySolution>> algorithm = new MOCHCBuilder(
            (BinaryProblem) problemList.get(i).getProblem())
            .setInitialConvergenceCount(0.25)
            .setConvergenceValue(3)
            .setPreservedPopulation(0.05)
            .setPopulationSize(100)
            .setMaxEvaluations(25000)
            .setCrossover(crossoverOperator)
            .setNewGenerationSelection(newGenerationSelection)
            .setCataclysmicMutation(mutationOperator)
            .setParentSelection(parentsSelection)
            .setEvaluator(new SequentialSolutionListEvaluator<BinarySolution>())
            .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }
    }
    return algorithms;
  }
159
function list(scriptModule, agent, msg, cb) {
    const servers = [];
    const scripts = [];
    const idMap = agent.idMap;

    for (const sid in idMap) {
        if (idMap.hasOwnProperty(sid)) {
            servers.push(sid);
        }
    }

    fs.readdir(scriptModule.root, (err, filenames) => {
        if (err) {
            filenames = [];
        }
        for (let i = 0, l = filenames.length; i < l; i++) {
            scripts.push(filenames[i]);
        }

        cb(null, {
            servers: servers,
            scripts: scripts
        });
    });
}
1258871
def setup(self):
        
        neb_dirs = self.neb_dirs

        if self.backup:
            # Back up KPOINTS, INCAR, POTCAR
            for f in VASP_NEB_INPUT_FILES:
                shutil.copy(f, "{}.orig".format(f))
            # Back up POSCARs
            for path in neb_dirs:
                poscar = os.path.join(path, "POSCAR")
                shutil.copy(poscar, "{}.orig".format(poscar))

        if self.half_kpts and os.path.exists("KPOINTS"):
            kpts = Kpoints.from_file("KPOINTS")
            kpts.kpts = np.maximum(np.array(kpts.kpts) / 2, 1)
            kpts.kpts = kpts.kpts.astype(int).tolist()
            if tuple(kpts.kpts[0]) == (1, 1, 1):
                kpt_dic = kpts.as_dict()
                kpt_dic["generation_style"] = 'Gamma'
                kpts = Kpoints.from_dict(kpt_dic)
            kpts.write_file("KPOINTS")

        if self.auto_npar:
            try:
                incar = Incar.from_file("INCAR")
                import multiprocessing
                # Try sge environment variable first
                # (since multiprocessing counts cores on the current
                # machine only)
                ncores = os.environ.get('NSLOTS') or multiprocessing.cpu_count()
                ncores = int(ncores)
                for npar in range(int(math.sqrt(ncores)),
                                  ncores):
                    if ncores % npar == 0:
                        incar["NPAR"] = npar
                        break
                incar.write_file("INCAR")
            except:
                pass

        if self.auto_continue and \
                os.path.exists("STOPCAR") and \
                not os.access("STOPCAR", os.W_OK):
            # Remove STOPCAR
            os.chmod("STOPCAR", 0o644)
            os.remove("STOPCAR")

            # Copy CONTCAR to POSCAR
            for path in self.neb_sub:
                contcar = os.path.join(path, "CONTCAR")
                poscar = os.path.join(path, "POSCAR")
                shutil.copy(contcar, poscar)

        if self.settings_override is not None:
            VaspModder().apply_actions(self.settings_override)
210096
def _setup_all_outputs(self) -> None:
        """"""

        # The way this is gonna work is that we assume an output should be set up iff it has a
        # credentials_ directory under our secrets dir.
        for key in self.outputs.keys():
            credentials_dir = path.join(self.secrets_dir, f"credentials_{key}")

            # special-case birdsite for historical reasons.
            if key == "birdsite" and not path.isdir(credentials_dir) \
                    and path.isfile(path.join(self.secrets_dir, "CONSUMER_KEY")):
                credentials_dir = self.secrets_dir

            if path.isdir(credentials_dir):
                output_skeleton = self.outputs[key]

                output_skeleton["active"] = True

                obj: Any = output_skeleton["obj"]
                obj.cred_init(secrets_dir=credentials_dir, log=self.log, bot_name=self.bot_name)

                output_skeleton["obj"] = obj

                self.outputs[key] = output_skeleton
192268
def _run_qsnp_paired(align_bams, items, ref_file, assoc_files,
                     region=None, out_file=None):
    
    config = items[0]["config"]
    if out_file is None:
        out_file = "%s-paired-variants.vcf" % os.path.splitext(align_bams[0])[0]
    if not utils.file_exists(out_file):
        out_file = out_file.replace(".gz", "")
        with file_transaction(config, out_file) as tx_out_file:
            with tx_tmpdir(config) as tmpdir:
                with utils.chdir(tmpdir):
                    paired = get_paired_bams(align_bams, items)
                    qsnp = config_utils.get_program("qsnp", config)
                    resources = config_utils.get_resources("qsnp", config)
                    mem = " ".join(resources.get("jvm_opts", ["-Xms750m -Xmx4g"]))
                    qsnp_log = os.path.join(tmpdir, "qsnp.log")
                    qsnp_init = os.path.join(tmpdir, "qsnp.ini")
                    if region:
                        paired = _create_bam_region(paired, region, tmpdir)
                    _create_input(paired, tx_out_file, ref_file, assoc_files['dbsnp'], qsnp_init)
                    cl = ("{qsnp} {mem} -i {qsnp_init} -log {qsnp_log}")
                    do.run(cl.format(**locals()), "Genotyping paired variants with Qsnp", {})
        out_file = _filter_vcf(out_file)
        out_file = bgzip_and_index(out_file, config)
    return out_file
1174595
def _varscan_work(align_bams, ref_file, items, target_regions, out_file):
    
    config = items[0]["config"]

    orig_out_file = out_file
    out_file = orig_out_file.replace(".vcf.gz", ".vcf")

    max_read_depth = "1000"
    sample_list = _create_sample_list(align_bams, out_file)
    mpileup = samtools.prep_mpileup(align_bams, ref_file, config, max_read_depth,
                                    target_regions=target_regions, want_bcf=False)
    # VarScan fails to generate a header on files that start with
    # zerocoverage calls; strip these with grep, we're not going to
    # call on them
    remove_zerocoverage = r"{ ifne grep -v -P '\t0\t\t$' || true; }"
    # we use ifne from moreutils to ensure we process only on files with input, skipping otherwise
    # http://manpages.ubuntu.com/manpages/natty/man1/ifne.1.html
    with tx_tmpdir(items[0]) as tmp_dir:
        jvm_opts = _get_jvm_opts(config, tmp_dir)
        opts = " ".join(_varscan_options_from_config(config))
        min_af = float(utils.get_in(config, ("algorithm", "min_allele_fraction"), 10)) / 100.0
        fix_ambig_ref = vcfutils.fix_ambiguous_cl()
        fix_ambig_alt = vcfutils.fix_ambiguous_cl(5)
        py_cl = os.path.join(os.path.dirname(sys.executable), "py")
        export = utils.local_path_export()
        cmd = ("{export} {mpileup} | {remove_zerocoverage} | "
               "ifne varscan {jvm_opts} mpileup2cns {opts} "
               "--vcf-sample-list {sample_list} --min-var-freq {min_af} --output-vcf --variants | "
               """{py_cl} -x 'bcbio.variation.vcfutils.add_contig_to_header(x, "{ref_file}")' | """
               "{py_cl} -x 'bcbio.variation.varscan.fix_varscan_output(x)' | "
               "{fix_ambig_ref} | {fix_ambig_alt} | ifne vcfuniqalleles > {out_file}")
        do.run(cmd.format(**locals()), "Varscan", None,
                [do.file_exists(out_file)])
    os.remove(sample_list)
    # VarScan can create completely empty files in regions without
    # variants, so we create a correctly formatted empty file
    if os.path.getsize(out_file) == 0:
        write_empty_vcf(out_file)

    if orig_out_file.endswith(".gz"):
        vcfutils.bgzip_and_index(out_file, config)
551654
def prep_system(run_info_yaml, bcbio_system=None):
    
    work_dir = os.getcwd()
    config, config_file = config_utils.load_system_config(bcbio_system, work_dir)
    dirs = setup_directories(work_dir, os.path.normpath(os.path.dirname(os.path.dirname(run_info_yaml))),
                             config, config_file)
    return [dirs, config, run_info_yaml]
118871
def _setup_fun_config(self, fun_conf):
        '''
        
        '''
        conf = copy.deepcopy(self.config)
        conf['file_client'] = 'local'
        conf['fun'] = ''
        conf['arg'] = []
        conf['kwarg'] = {}
        conf['cache_jobs'] = False
        conf['print_metadata'] = False
        conf.update(fun_conf)
        conf['fun'] = conf['fun'].split(':')[-1]  # Discard typing prefix

        return conf
428434
def run(align_bams, items, ref_file, assoc_files, region=None, out_file=None):
    
    paired = vcfutils.get_paired_bams(align_bams, items)
    assert paired and not paired.normal_bam, ("smCounter2 supports tumor-only variant calling: %s" %
                                              (",".join([dd.get_sample_name(d) for d in items])))
    vrs = bedutils.population_variant_regions(items)
    target = shared.subset_variant_regions(vrs, region,
                                            out_file, items=items, do_merge=True)
    out_file = out_file.replace(".vcf.gz", ".vcf")
    out_prefix = utils.splitext_plus(os.path.basename(out_file))[0]
    if not utils.file_exists(out_file) and not utils.file_exists(out_file + ".gz"):
        with file_transaction(paired.tumor_data, out_file) as tx_out_file:
            cmd = ["smCounter2", "--runPath", os.path.dirname(tx_out_file),
                   "--outPrefix", out_prefix,
                   "--bedTarget", target, "--refGenome", ref_file,
                   "--bamFile", paired.tumor_bam, "--bamType", "consensus",
                   "--nCPU", dd.get_num_cores(paired.tumor_data)]
            do.run(cmd, "smcounter2 variant calling")
            for fname in glob.glob(os.path.join(os.path.dirname(tx_out_file), "*.smCounter*")):
                shutil.move(fname, os.path.join(os.path.dirname(out_file), os.path.basename(fname)))
            utils.symlink_plus(os.path.join(os.path.dirname(out_file),
                                            "%s.smCounter.cut.vcf" % out_prefix),
                               out_file)
    return vcfutils.bgzip_and_index(out_file, paired.tumor_data["config"], remove_orig=False,
                                    prep_cmd="sed 's#FORMAT\t%s#FORMAT\t%s#' | %s" %
                                    (out_prefix, dd.get_sample_name(paired.tumor_data),
                                     vcfutils.add_contig_to_header_cl(dd.get_ref_file(paired.tumor_data), out_file)))