Retrieve the qseq directory within Solexa flowcell output.
504902
def get_qseq_dir(fc_dir):
    
    machine_bc = os.path.join(fc_dir, "Data", "Intensities", "BaseCalls")
    if os.path.exists(machine_bc):
        return machine_bc
    # otherwise assume we are in the qseq directory
    # XXX What other cases can we end up with here?
    else:
        return fc_dir
91281
def getCol(self, x):
        """"""
        return [self.getCell(x, i) for i in self.__size_range]
1022657
def summary(*samples):
    """"""
    samples = list(utils.flatten(samples))
    work_dir = dd.get_work_dir(samples[0])
    multiqc = config_utils.get_program("multiqc", samples[0]["config"])
    if not multiqc:
        logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
    out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "multiqc"))
    out_data = os.path.join(out_dir, "multiqc_data")
    out_file = os.path.join(out_dir, "multiqc_report.html")
    file_list = os.path.join(out_dir, "list_files.txt")
    work_samples = cwlutils.unpack_tarballs([utils.deepish_copy(x) for x in samples], samples[0])
    work_samples = _summarize_inputs(work_samples, out_dir)
    if not utils.file_exists(out_file):
        with tx_tmpdir(samples[0], work_dir) as tx_out:
            in_files = _get_input_files(work_samples, out_dir, tx_out)
            in_files += _merge_metrics(work_samples, out_dir)
            if _one_exists(in_files):
                with utils.chdir(out_dir):
                    _create_config_file(out_dir, work_samples)
                    input_list_file = _create_list_file(in_files, file_list)
                    if dd.get_tmp_dir(samples[0]):
                        export_tmp = "export TMPDIR=%s && " % dd.get_tmp_dir(samples[0])
                    else:
                        export_tmp = ""
                    locale_export = utils.locale_export()
                    path_export = utils.local_path_export()
                    other_opts = config_utils.get_resources("multiqc", samples[0]["config"]).get("options", [])
                    other_opts = " ".join([str(x) for x in other_opts])
                    cmd = ("{path_export}{export_tmp}{locale_export} "
                           "{multiqc} -f -l {input_list_file} {other_opts} -o {tx_out}")
                    do.run(cmd.format(**locals()), "Run multiqc")
                    if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
                        shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
                        shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
    samples = _group_by_sample_and_batch(samples)
    if utils.file_exists(out_file) and samples:
        data_files = set()
        for i, data in enumerate(samples):
            data_files.add(os.path.join(out_dir, "report", "metrics", dd.get_sample_name(data) + "_bcbio.txt"))
        data_files.add(os.path.join(out_dir, "report", "metrics", "target_info.yaml"))
        data_files.add(os.path.join(out_dir, "multiqc_config.yaml"))
        [data_files.add(f) for f in glob.glob(os.path.join(out_dir, "multiqc_data", "*"))]
        data_files = [f for f in data_files if f and utils.file_exists(f)]
        if "summary" not in samples[0]:
            samples[0]["summary"] = {}
        samples[0]["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}

        data_json = os.path.join(out_dir, "multiqc_data", "multiqc_data.json")
        data_json_final = _save_uploaded_data_json(samples, data_json, os.path.join(out_dir, "multiqc_data"))
        if data_json_final:
            samples[0]["summary"]["multiqc"]["secondary"].append(data_json_final)

        # Prepare final file list and inputs for downstream usage
        file_list_final = _save_uploaded_file_list(samples, file_list, out_dir)
        if file_list_final:
            samples[0]["summary"]["multiqc"]["secondary"].append(file_list_final)
            if any([cwlutils.is_cwl_run(d) for d in samples]):
                for indir in ["inputs", "report"]:
                    tarball = os.path.join(out_dir, "multiqc-%s.tar.gz" % (indir))
                    if not utils.file_exists(tarball):
                        with utils.chdir(out_dir):
                            cmd = ["tar", "-czvpf", tarball, indir]
                            do.run(cmd, "Compress multiqc inputs: %s" % indir)
                    samples[0]["summary"]["multiqc"]["secondary"].append(tarball)

    if any([cwlutils.is_cwl_run(d) for d in samples]):
        samples = _add_versions(samples)

    return [[data] for data in samples]
228358
def _get_seqtype_from_ext(handle):
    ''''''
    if isinstance(handle, basestring):
        name = handle
    elif hasattr(handle, 'filename'):
        name = handle.filename
    elif hasattr(handle, 'name'):
        name = handle.name
    else:
        raise ValueError("Unknown datatype for handle!")

    modifier = ''
    dummy, ext = path.splitext(name.lower())
    if ext == ".gz":
        modifier = 'gz-'
        dummy, ext = path.splitext(dummy)

    if not ext:
        ext = "." + dummy

    if ext in (".gbk", ".gb", ".genbank", ".gbff"):
        return modifier + "genbank"
    elif ext in (".embl", ".emb"):
        return modifier + "embl"
    elif ext in (".fa", ".fasta", ".fna", ".faa", ".fas"):
        return modifier + "fasta"
    else:
        raise ValueError("Unknown file format '%s'." % ext)
1247089
def _get_data_path(data_dir=None):
    
    data_dir = data_dir or ''

    default_dirs = [Path(d).expanduser().resolve()
                    for d in os.getenv('CRN_SHARED_DATA', '').split(os.pathsep)
                    if d.strip()]
    default_dirs += [Path(d).expanduser().resolve()
                     for d in os.getenv('CRN_DATA', '').split(os.pathsep)
                     if d.strip()]
    default_dirs += [NIWORKFLOWS_CACHE_DIR]

    return [Path(d).expanduser()
            for d in data_dir.split(os.pathsep) if d.strip()] or default_dirs
365232
def fastqmover(self):
        """"""
        # Create the sequence path if necessary
        make_path(self.sequencepath)
        # Iterate through all the sample names
        for sample in self.metadata.samples:
            # Make directory variables
            outputdir = os.path.join(self.sequencepath, sample.name)
            # Demultiplexed files will be present in the project name subfolder within the fastq destination folder
            if self.demultiplex:
                glob_dir = self.projectpath
            # Undemultiplexed reads (Undetermined_S0_R1_001.fastq.gz) are present in the fastq destination folder
            else:
                glob_dir = self.fastqdestination
            # Sometimes the files are put in self.fastqdestination rather than self.projectpath? Matt has this problem
            # and I don't understands what's going on
            if not os.path.isdir(glob_dir):
                glob_dir = os.path.dirname(glob_dir)
            # Glob all the .gz files in the subfolders - projectpath/Sample_:sample.name/*.gz
            for fastq in sorted(glob(os.path.join(glob_dir, '*.gz'))):
                fastqname = os.path.basename(fastq)
                # Set the name of the destination file
                outputfile = os.path.join(self.sequencepath, fastqname)
                # Link the file if it doesn't already exist
                if not os.path.isfile(outputfile):
                    relative_symlink(src_file=fastq,
                                     output_dir=self.sequencepath)
            # Repopulate .strainfastqfiles with the freshly-linked/copied files
            if self.demultiplex:
                fastqfiles = glob(os.path.join(self.sequencepath, '{}*.fastq*'.format(sample.name)))
                fastqfiles = sorted([fastq for fastq in fastqfiles if 'trimmed' not in os.path.basename(fastq)])
            # Undemultiplexed files will not have the sample name in the file name
            else:
                fastqfiles = sorted(glob(os.path.join(glob_dir, '*.gz')))
            # Populate the metadata object with the name/path of the fastq files
            sample.general.fastqfiles = fastqfiles
            # Save the outputdir to the metadata object
            sample.run.outputdirectory = outputdir
            sample.general.outputdirectory = outputdir
            sample.general.bestassemblyfile = True
            sample.general.trimmedcorrectedfastqfiles = sorted(sample.general.fastqfiles)
            sample.general.logout = os.path.join(sample.general.outputdirectory, 'logout')
            sample.general.logerr = os.path.join(sample.general.outputdirectory, 'logerr')
            sample.commands = GenObject()
466499
def fastqc(self):
        """"""
        while True:  # while daemon
            threadlock = threading.Lock()
            # Unpack the variables from the queue
            (sample, systemcall, outputdir, fastqcreads) = self.qcqueue.get()
            # Check to see if the output HTML file already exists
            try:
                _ = glob(os.path.join(outputdir, '*.html'))[0]
            except IndexError:
                # Make the output directory
                make_path(outputdir)
                # Run the system calls
                outstr = str()
                errstr = str()
                out, err = run_subprocess(systemcall)
                outstr += out
                errstr += err
                out, err = run_subprocess(fastqcreads)
                outstr += out
                errstr += err
                # Acquire thread lock, and write the logs to file
                threadlock.acquire()
                write_to_logfile(systemcall, systemcall, self.logfile, sample.general.logout, sample.general.logerr,
                                 None, None)
                write_to_logfile(fastqcreads, fastqcreads, self.logfile, sample.general.logout, sample.general.logerr,
                                 None, None)
                write_to_logfile(outstr, errstr, self.logfile, sample.general.logout, sample.general.logerr, None, None)
                threadlock.release()
                # Rename the outputs
                try:
                    shutil.move(os.path.join(outputdir, 'stdin_fastqc.html'),
                                os.path.join(outputdir, '{}_fastqc.html'.format(sample.name)))
                    shutil.move(os.path.join(outputdir, 'stdin_fastqc.zip'),
                                os.path.join(outputdir, '{}_fastqc.zip'.format(sample.name)))
                except IOError:
                    pass
            # Signal to qcqueue that job is done
            self.qcqueue.task_done()
954660
def blast_pdb(self, seq_ident_cutoff=0, evalue=0.0001, display_link=False,
                  outdir=None, force_rerun=False):
        """"""
        if not outdir:
            outdir = self.sequence_dir
            if not outdir:
                raise ValueError('Output directory must be specified')

        if not self.seq_str:
            log.error('{}: no sequence loaded'.format(self.id))
            return None

        try:
            blast_results = ssbio.databases.pdb_seq.blast_pdb(self.seq_str,
                                                              outfile='{}_blast_pdb.xml'.format(custom_slugify(self.id)),
                                                              outdir=outdir,
                                                              force_rerun=force_rerun,
                                                              evalue=evalue,
                                                              seq_ident_cutoff=seq_ident_cutoff,
                                                              link=display_link)
        except requests.ConnectionError as e:
            log.error('{}: BLAST request timed out'.format(self.id))
            print(e)
            return None

        return blast_results
945091
def askForFolder(parent, msg = None):
    '''
    
    '''
    msg = msg or 'Select folder'
    caller = _callerName().split(".")
    name = "/".join([LAST_PATH, caller[-1]])
    namespace = caller[0]
    path = pluginSetting(name, namespace)
    folder =  QtWidgets.QFileDialog.getExistingDirectory(parent, msg, path)
    if folder:
        setPluginSetting(name, folder, namespace)
    return folder
195620
def load(self, binary: pyquil.Program) -> 'QuantumFlowQVM':
        

        assert self.status in ['connected', 'done']
        prog = quil_to_program(str(binary))

        self._prog = prog
        self.program = binary
        self.status = 'loaded'

        return self
987063
def name_file(lane: int, flowcell: str, sample: str, read: int,
                  undetermined: bool=False, date: dt.datetime=None, index: str=None) -> str:
        """"""
        flowcell = f"{flowcell}-undetermined" if undetermined else flowcell
        date_str = date.strftime('%y%m%d') if date else '171015'
        index = index if index else 'XXXXXX'
        return f"{lane}_{date_str}_{flowcell}_{sample}_{index}_{read}.fastq.gz"
782228
def ReadFlowOutputPluginLogEntries(self,
                                     client_id,
                                     flow_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """"""
    entries = sorted(
        self.flow_output_plugin_log_entries.get((client_id, flow_id), []),
        key=lambda e: e.timestamp)

    entries = [e for e in entries if e.output_plugin_id == output_plugin_id]

    if with_type is not None:
      entries = [e for e in entries if e.log_entry_type == with_type]

    return entries[offset:offset + count]
1084862
def _cutadapt_se_cmd(fastq_files, out_files, base_cmd, data):
    
    min_length = dd.get_min_read_length(data)
    cmd = base_cmd + " --minimum-length={min_length} ".format(**locals())
    fq1 = objectstore.cl_input(fastq_files[0])
    of1 = out_files[0]
    cmd += " -o {of1_tx} " + str(fq1)
    cmd = "%s | tee > {log_tx}" % cmd
    return cmd
110871
def goToDirectory(alias):
	""""""
	if not settings.platformCompatible():
		return False
	data = pickle.load(open(settings.getDataFile(), "rb"))
	try:
		data[alias]
	except KeyError:
		speech.fail("Sorry, it doesn't look like you have saved " + alias + " yet.")
		speech.fail("Go to the directory you'd like to save and type 'hallie save as " + alias + "\'")
		return
	try:
		(output, error) = subprocess.Popen(["osascript", "-e", CHANGE_DIR % (data[alias])], stdout=subprocess.PIPE).communicate()
	except:
		speech.fail("Something seems to have gone wrong. Please report this error to michaelmelchione@gmail.com.")
		return
	speech.success("Successfully navigating to " + data[alias])
504548
def get_runtime_path(self, filename=None):
        """"""
        return self.get_path(prefix=settings.FLOW_EXECUTOR['RUNTIME_DIR'], filename=filename)
904332
def get_output(self):
    
    if self.__dax:
      # we are a dax running in grid mode so we need to resolve the
      # frame file metadata into LFNs so pegasus can query the RLS
      if self.__lfn_list is None:

        if self.job().lsync_cache():
          # get the lfns from the lsync cache object
          if self.__lfn_list is None:
            self.__lfn_list = self.job().lsync_cache().get_lfns(
              self.get_observatory(), self.get_type(),
              self.get_start(), self.get_end())

        else:
          # try querying the ligo_data_find server
          try:
            server = os.environ['LIGO_DATAFIND_SERVER']
          except KeyError:
            raise RuntimeError, \
              "Environment variable LIGO_DATAFIND_SERVER is not set"

          try:
            h = httplib.HTTPConnection(server)
          except:
            # try and get a proxy or certificate
            # FIXME this doesn't check that it is valid, though
            cert = None
            key = None
            try:
              proxy = os.environ['X509_USER_PROXY']
              cert = proxy
              key = proxy
            except:
              try:
                cert = os.environ['X509_USER_CERT']
                key = os.environ['X509_USER_KEY']
              except:
                uid = os.getuid()
                proxy_path = "/tmp/x509up_u%d" % uid
                if os.access(path, os.R_OK):
                  cert = proxy_path
                  key = proxy_path

            h = httplib.HTTPSConnection(server, key_file = key, cert_file = cert)

          # construct the URL for a simple data find query
          url = "/LDR/services/data/v1/gwf/%s/%s/%s,%s.json" % (
            self.get_observatory(), self.get_type(),
            str(self.get_start()), str(self.get_end()))

          # query the server
          h.request("GET", url)
          response = h.getresponse()

          if response.status != 200:
            msg = "Server returned code %d: %s" % (response.status, response.reason)
            body = response.read()
            msg += body
            raise RuntimeError, msg

          # since status is 200 OK read the URLs
          body = response.read()

          # decode the JSON
          urlList = cjson.decode(body)
          lfnDict = {}
          for url in urlList:
            path = urlparse.urlparse(url)[2]
            lfn = os.path.split(path)[1]
            lfnDict[lfn] = 1

          self.__lfn_list = lfnDict.keys()
          self.__lfn_list.sort()

      return self.__lfn_list
    else:
      return self.__output
370111
def filer(self):
        
        # Get a list of all the .fastq files in the sequence path
        fastq = sorted(glob(os.path.join(self.sequencepath, '*fastq*')))
        # Initialise a list to store .fasta formatted files
        fasta = list()
        # Get all the files in the sequence path
        for sequencefile in glob(os.path.join(self.sequencepath, '*')):
            # Split the extension from the file
            extension = os.path.splitext(sequencefile)[1]
            # If the extension is one of the acceptable FASTA extensions, add it to the list of FASTA files
            if extension in self.extensions:
                fasta.append(sequencefile)
        # Use filer from accessoryFunctions to find the base file name of each .fastq file/pair
        fastqnames = filer(fastq)
        # Get all the .fastq files into metadata objects
        for fastqname in sorted(fastqnames):
            # Create the metadata object
            sample = MetadataObject()
            sample.name = os.path.basename(fastqname)
            # Create an attribute for the current analysis
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, fastqname)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Get the fastq files specific to the fastqname
            specificfastq = glob(os.path.join(self.sequencepath, '{fq}*.fastq*'.format(fq=fastqname)))
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fastq'
            # Link the files to the output folder
            for fastq in specificfastq:
                relative_symlink(src_file=fastq,
                                 output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in
                                         glob(os.path.join(sample[self.analysistype].outputdir, '{fq}*.fastq*'
                                                           .format(fq=fastqname))) if 'trimmed' not in fastq]
            # Populate certain attributes in order to be compatible with other local software
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            # Append the metadata to the list of samples
            self.metadata.append(sample)
        # Add any .fasta formatted files to the metadata object
        for fastafile in sorted(fasta):
            sample = MetadataObject()
            sample.name = os.path.splitext(os.path.split(fastafile)[-1])[0]
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, 'detailed_reports', sample.name)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fasta'
            # Link the .fasta files to :self.path/:filename
            relative_symlink(src_file=fastafile,
                             output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in glob(os.path.join(sample[self.analysistype].outputdir,
                                                                              '{sn}*{ext}*'
                                                                              .format(
                                                                                  sn=sample.name,
                                                                                  ext=os.path.splitext(fastafile)[1])))]
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            sample[self.analysistype].assemblyfile = fastafile
            # Append the metadata to the list of samples
            self.metadata.append(sample)