Parse run files generated by Cluster Flow.
        Currently gets pipeline IDs and associated steps.
13614
def parse_clusterflow_runfiles(self, f):
        

        data = dict()
        in_comment = False
        seen_pipeline = False
        cf_file = False
        for l in f['f']:
            l = l.rstrip()
            # Check that this is from Cluster Flow
            if 'Cluster Flow' in l:
                cf_file = True
            # Header
            if l.startswith('Pipeline: '):
                data['pipeline_name'] = l[10:]
            if l.startswith('Pipeline ID: '):
                data['pipeline_id'] = l[13:]
            if l.startswith('Created at '):
                data['pipeline_start'] = l[11:]
            # Config settings
            if l.startswith('@'):
                s = l.split(None, 1)
                key = s[0].replace('@', '').strip()
                try:
                    data[key] = "\t".join(s[1:])
                except IndexError:
                    data[key] = True
            # Comments
            if l.startswith('/*'):
                in_comment = True
            if l.startswith('*/'):
                in_comment = False
            if in_comment:
                if 'comment' not in data:
                    data['comment'] = ''
                data['comment'] += l+"\n"
            # Pipeline steps
            if l.strip().startswith('#'):
                if 'pipeline_steps' not in data:
                    data['pipeline_steps'] = []
                data['pipeline_steps'].append(l)
                seen_pipeline = True
            # Step output files
            elif seen_pipeline:
                s = l.split("\t")
                if len(s) > 1:
                    if 'files' not in data:
                        data['files'] = OrderedDict()
                    if s[0] not in data['files']:
                        data['files'][s[0]] = []
                    data['files'][s[0]].append(s[1:])
        # Parse the start date
        dt = None
        if 'pipeline_id' in data:
            s = data['pipeline_id'].split('_')
            dt = datetime.datetime.fromtimestamp(int(s[-1]))
        elif 'pipeline_start' in data:
            dt_r = re.match(r'(\d{2}):(\d{2}), (\d{2})-(\d{2})-(\d{4})', data['pipeline_start'])
            if dt_r:
                dt = datetime.datetime(
                    int(dt_r.group(5)), # year
                    int(dt_r.group(4)), # month
                    int(dt_r.group(3)), # day
                    int(dt_r.group(1)), # hour
                    int(dt_r.group(2))  # minute
                )

        # Not a Cluster Flow file (eg. Nextflow .run file)
        if not cf_file:
            return None

        if dt is not None:
            data['pipeline_start_dateparts'] = {
                'year':        dt.year,
                'month':       dt.month,
                'day':         dt.day,
                'hour':        dt.hour,
                'minute':      dt.minute,
                'second':      dt.second,
                'microsecond': dt.microsecond,
                'timestamp':   time.mktime(dt.timetuple())
            }
        # Cluster Flow v0.4 and before did not print the pipeline ID in run files
        # Try to guess - will be wrong as no microsecond info, but hopefully unique
        # and reproducible for other run files
        if 'pipeline_id' not in data:
            if 'pipeline_name' in data and 'pipeline_start_dateparts' in data:
                log.debug('Trying to guess pipeline ID for file "{}"'.format(f['fn']))
                data['pipeline_id'] = 'cf_{}_{}'.format(data['pipeline_name'], data['pipeline_start_dateparts']['timestamp'])
        return data
1026118
def run(self):
        
        try:
            self.loader.find_and_load_step_definitions()
        except StepLoadingError, e:
            print "Error loading step definitions:\n", e
            return

        results = []
        if self.explicit_features:
            features_files = self.explicit_features
        else:
            features_files = self.loader.find_feature_files()
        if self.random:
            random.shuffle(features_files)

        if not features_files:
            self.output.print_no_features_found(self.loader.base_dir)
            return

        processes = Pool(processes=self.parallelization)
        test_results_it = processes.imap_unordered(
            worker_process, [(self, filename) for filename in features_files]
        )
        
        all_total = ParallelTotalResult()
        for result in test_results_it:
            all_total += result['total']
            sys.stdout.write(result['stdout'])
            sys.stderr.write(result['stderr'])

        return all_total
1143770
def _build_process_container_tree(self, pids):
        
        containers = []
        procs = []
        ppids = []
        childs = {}
        for pid in pids:
            proc = process.Process(pid)
            procs.append(proc)
            ppids.append(proc.ppid)
            if proc.ppid not in childs:
                childs[proc.ppid] = []
            childs[proc.ppid].append(proc)
        ppids = set(ppids)
        tops = [proc for proc in procs if proc.ppid not in pids]
        if len(tops) == 0:
            tops = procs

        def build_tree(proc_list):
            _containers = []
            for proc in proc_list:
                if not self.args.show_kthread and proc.is_kthread():
                    continue

                cont = TreeContainer(proc)

                if proc.pid in childs:
                    cont.childs = build_tree(childs[proc.pid])
                _containers.append(cont)
            return _containers

        for top_proc in tops:
            if not self.args.show_kthread and top_proc.is_kthread():
                continue

            cont = TreeContainer(top_proc)
            if top_proc.pid in childs:
                cont.childs = build_tree(childs[top_proc.pid])
            containers.append(cont)

        return containers
1088861
def process(self, t_path, name_info, i_str):
        '''

        '''
        def cids_and_fcs():
            count = 0
            seen = set()
            for si in Chunk(t_path):
                clean_html = getattr(si.body, 'clean_html', '')
                if clean_html is None or len(clean_html.strip()) == 0:
                    logger.warn('dropping SI lacking clean_html: %r', si.abs_url)
                    continue
                if 'other_features' in si.other_content:
                    other_features = json.loads(si.other_content['other_features'].raw)
                else:
                    other_features = None
                fc = html_to_fc(
                    clean_html=si.body.clean_html.decode('utf-8'),
                    clean_visible=si.body.clean_visible.decode('utf-8'),
                    encoding='utf-8',
                    url=si.abs_url,
                    timestamp=si.stream_time.epoch_ticks,
                    other_features=other_features,
                )
                add_sip_to_fc(fc, self.tfidf)
                content_id = mk_content_id(str(fc.get(u'meta_url')))
                if content_id in seen:
                    logger.warn('dropping duplicate content_id=%r', content_id)
                else:
                    seen.add(content_id)
                    yield content_id, fc
                    count += 1
            logger.info('saved %d FCs from %d SIs', count, len(seen))
        self.store.put(cids_and_fcs())
        ## interface spec of streamcorpus_pipeline writers requires
        ## returning a list of locally generated paths.
        return []
444501
def get_running_jobs(opts):
    '''
    
    '''

    ret = []
    proc_dir = os.path.join(opts['cachedir'], 'proc')
    if not os.path.isdir(proc_dir):
        return ret
    for fn_ in os.listdir(proc_dir):
        path = os.path.join(proc_dir, fn_)
        data = read_proc_file(path, opts)
        if not data:
            continue
        if not is_pid_healthy(data['pid']):
            continue
        ret.append(data)
    return ret
456269
def _get_input_files(samples, base_dir, tx_out_dir):
    
    in_files = collections.defaultdict(list)
    for data in samples:
        sum_qc = tz.get_in(["summary", "qc"], data, {})
        if sum_qc in [None, "None"]:
            sum_qc = {}
        elif isinstance(sum_qc, six.string_types):
            sum_qc = {dd.get_algorithm_qc(data)[0]: sum_qc}
        elif not isinstance(sum_qc, dict):
            raise ValueError("Unexpected summary qc: %s" % sum_qc)
        for program, pfiles in sum_qc.items():
            if isinstance(pfiles, dict):
                pfiles = [pfiles["base"]] + pfiles.get("secondary", [])
            # CWL: presents output files as single file plus associated secondary files
            elif isinstance(pfiles, six.string_types):
                if os.path.exists(pfiles):
                    pfiles = [os.path.join(basedir, f) for basedir, subdir, filenames in os.walk(os.path.dirname(pfiles)) for f in filenames]
                else:
                    pfiles = []
            in_files[(dd.get_sample_name(data), program)].extend(pfiles)
    staged_files = []
    for (sample, program), files in in_files.items():
        cur_dir = utils.safe_makedir(os.path.join(base_dir, "inputs", sample, program))
        for f in files:
            if _check_multiqc_input(f) and _is_good_file_for_multiqc(f):
                if _in_temp_directory(f) or any([cwlutils.is_cwl_run(d) for d in samples]):
                    staged_f = os.path.join(cur_dir, os.path.basename(f))
                    shutil.copy(f, staged_f)
                    staged_files.append(staged_f)
                else:
                    staged_files.append(f)
    staged_files.extend(get_qsig_multiqc_files(samples))
    # Back compatible -- to migrate to explicit specifications in input YAML
    if not any([cwlutils.is_cwl_run(d) for d in samples]):
        staged_files += ["trimmed", "htseq-count/*summary"]
        # Add in created target_info file
        if os.path.isfile(os.path.join(base_dir, "report", "metrics", "target_info.yaml")):
            staged_files += [os.path.join(base_dir, "report", "metrics", "target_info.yaml")]
    return sorted(list(set(staged_files)))
547217
def runner(self):
        
        logging.info('Starting {} analysis pipeline'.format(self.analysistype))
        if not self.pipeline:
            # If the metadata has been passed from the method script, self.pipeline must still be false in order to
            # get Sippr() to function correctly, but the metadata shouldn't be recreated
            try:
                _ = vars(self.runmetadata)['samples']
            except AttributeError:
                # Create the objects to be used in the analyses
                objects = Objectprep(self)
                objects.objectprep()
                self.runmetadata = objects.samples

        else:
            for sample in self.runmetadata.samples:
                setattr(sample, self.analysistype, GenObject())
                sample.run.outputdirectory = sample.general.outputdirectory
        self.threads = int(self.cpus / len(self.runmetadata.samples)) \
            if self.cpus / len(self.runmetadata.samples) > 1 \
            else 1
        # Use a custom sippr method to use the full reference database as bait, and run mirabait against the FASTQ
        # reads - do not perform reference mapping yet
        SixteenSBait(self, self.cutoff)
        # Subsample 1000 reads from the FASTQ files
        self.subsample()
        # Convert the subsampled FASTQ files to FASTA format
        self.fasta()
        # Create BLAST databases if required
        self.makeblastdb()
        # Run BLAST analyses of the subsampled FASTA files against the NCBI 16S reference database
        self.blast()
        # Parse the BLAST results
        self.blastparse()
        # Feed the BLAST results into a modified sippr method to perform reference mapping using the calculated
        # genus of the sample as the mapping file
        SixteenSSipper(self, self.cutoff)
        # Create reports
        self.reporter()
667177
def read(self, path, ext=None, start=None, stop=None, recursive=False, npartitions=None):
        
        path = uri_to_path(path)
        files = self.list(path, ext=ext, start=start, stop=stop, recursive=recursive)

        nfiles = len(files)
        self.nfiles = nfiles

        if spark and isinstance(self.engine, spark):
            npartitions = min(npartitions, nfiles) if npartitions else nfiles
            rdd = self.engine.parallelize(enumerate(files), npartitions)
            return rdd.map(lambda kv: (kv[0], readlocal(kv[1]), kv[1]))
        else:
            return [(k, readlocal(v), v) for k, v in enumerate(files)]
1019781
def _pipeline_objects(pipeline_objects_from_pillars, pipeline_object_overrides):
    '''
    
    '''
    from_pillars = copy.deepcopy(__salt__['pillar.get'](pipeline_objects_from_pillars))
    from_pillars.update(pipeline_object_overrides)
    pipeline_objects = _standardize(_dict_to_list_ids(from_pillars))
    for pipeline_object in pipeline_objects:
        pipeline_object['fields'] = _properties_from_dict(pipeline_object['fields'])
    return pipeline_objects
319624
def clusterflow_pipelines_section(self):
        
        data = dict()
        pids_guessed = ''
        for f,d in self.clusterflow_runfiles.items():
            pid = d.get('pipeline_id', 'unknown')
            if d.get('pipeline_id_guess', False) is True:
                pid += '*'
                pids_guessed = ' Project IDs with an asterisk may be inaccurate.'
            # Count the number of files going into the first module
            num_starting_files = 0
            for step_name, files in d.get('files',{}).items():
                if step_name.startswith('start'):
                    num_starting_files += len(files)
            # Reformat the date so that column sorting works nicely
            if 'pipeline_start_dateparts' in d:
                dt = d['pipeline_start_dateparts']
                d['pipeline_start'] = '{}-{:02d}-{:02d} {:02d}:{:02d}'.format(dt['year'], dt['month'], dt['day'], dt['hour'], dt['minute'])
            if pid not in data:
                data[pid] = d
                data[pid]['num_starting_files'] = int(num_starting_files)
            else:
                data[pid]['num_starting_files'] += int(num_starting_files)

        headers = OrderedDict()
        headers['pipeline_name'] = {'title': 'Pipeline Name'}
        headers['pipeline_start'] = {'title': 'Date Started', 'description': 'Date and time that pipeline was started (YYYY-MM-DD HH:SS)'}
        headers['genome'] = {'title': 'Genome ID', 'description': 'ID of reference genome used'}
        headers['num_starting_files'] = {'title': '# Starting Files', 'format': '{:,.0f}', 'description': 'Number of input files at start of pipeline run.'}
        table_config = {
            'namespace': 'Cluster Flow',
            'id': 'clusterflow-pipelines-table',
            'table_title': 'Cluster Flow Pipelines',
            'col1_header': 'Pipeline ID',
            'no_beeswarm': True,
            'save_file': True
        }
        self.add_section (
            name = 'Pipelines',
            anchor = 'clusterflow-pipelines',
            description = 'Information about pipelines is parsed from <code>*.run</code> files. {}'.format(pids_guessed),
            plot = table.plot(data, headers, table_config),
            content = self.clusterflow_pipelines_printout()
        )
732951
def split_sequence_file_on_sample_ids_to_files(seqs,
                                               outdir):
    
    logger = logging.getLogger(__name__)
    logger.info('split_sequence_file_on_sample_ids_to_files'
                ' for file %s into dir %s' % (seqs, outdir))

    outputs = {}

    for bits in sequence_generator(seqs):
        sample = sample_id_from_read_id(bits[0])

        if sample not in outputs:
            outputs[sample] = open(join(outdir, sample + '.fasta'), 'w')
        outputs[sample].write(">%s\n%s\n" % (bits[0], bits[1]))
    for sample in outputs:
        outputs[sample].close()
    logger.info('split to %d files' % len(outputs))
234405
def Run(self, args):
    """"""
    try:
      directory = vfs.VFSOpen(args.pathspec, progress_callback=self.Progress)
    except (IOError, OSError) as e:
      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.IOERROR, e)
      return

    files = list(directory.ListFiles())
    files.sort(key=lambda x: x.pathspec.path)

    for response in files:
      self.SendReply(response)
734237
def run(self, redirects = []):
        """"""
        if not isinstance(redirects, redir.Redirects):
            redirects = redir.Redirects(self._env._redirects, *redirects)
        with copy.copy_session() as sess:
            self = copy.deepcopy(self)
            processes = self._run(redirects, sess)
        pipeline = RunningPipeline(processes, self)
        self._env.last_pipeline = pipeline
        return pipeline
273797
public String getRunningFlowIds() {
    final List<Integer> allIds = new ArrayList<>();
    getRunningFlowsIdsHelper(allIds, this.queuedFlows.getAllEntries());
    getRunningFlowsIdsHelper(allIds, this.runningExecutions.get().values());
    Collections.sort(allIds);
    return allIds.toString();
  }
513323
def cluster(args):
    

    args = _check_args(args)
    read_stats_file = op.join(args.dir_out, "read_stats.tsv")
    if file_exists(read_stats_file):
        os.remove(read_stats_file)

    bam_file, seq_obj = _clean_alignment(args)

    logger.info("Parsing matrix file")
    seqL, y, l = parse_ma_file(seq_obj, args.ffile)
    # y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'aligned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    if len(seqL.keys()) < 10:
        logger.error("It seems you have low coverage. Please check your fastq files have enough sequences.")
        raise ValueError("So few sequences.")

    logger.info("Cleaning bam file")
    y, l = _total_counts(seqL.keys(), seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'cleaned'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    clusL = _create_clusters(seqL, bam_file, args)
    y, l = _total_counts(clusL.seq.keys(), clusL.seq, aligned=True)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'clusters'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')

    logger.info("Solving multi-mapping events in the network of clusters")
    clusLred = _cleaning(clusL, args.dir_out)
    y, l = _total_counts(clusLred.clus, seqL)
    logger.info("counts after: %s" % sum(y.values()))
    logger.info("# sequences after: %s" % l)
    dt = pd.DataFrame({'sample': y.keys(), 'counts': y.values()})
    dt['step'] = 'meta-cluster'
    dt.to_csv(read_stats_file, sep="\t", index=False, header=False, mode='a')
    logger.info("Clusters up to %s" % (len(clusLred.clus.keys())))

    if args.show:
        logger.info("Creating sequences alignment to precursor")
        clusLred = show_seq(clusLred, args.index)

    clusLred = peak_calling(clusLred)

    clusLred = _annotate(args, clusLred)

    logger.info("Creating json and count matrix")
    json_file = _create_json(clusLred, args)

    logger.info("Output file in: %s" % args.dir_out)

    if args.db:
        name = args.db + ".db"
        logger.info("Create database: database/" + name)
        data = load_data(json_file)
        out_dir = op.join(args.dir_out, "database")
        make_database(data, name, out_dir)
    logger.info("Finished")
452576
def get_job_logs_from_workflow(workflow_id):
    """"""
    query_result = (
        db.session.query(
            models.CrawlerJob.logs,
        )
        .join(
            models.CrawlerWorkflowObject,
            models.CrawlerJob.job_id == models.CrawlerWorkflowObject.job_id,
        )
        .filter(models.CrawlerWorkflowObject.object_id == workflow_id)
        .one_or_none()
    )

    if query_result is None:
        click.secho(
            (
                "Workflow %s was not found, maybe it's not a crawl workflow?" %
                workflow_id
            ),
            fg='yellow',
        )
        sys.exit(1)

    _show_file(
        file_path=query_result[0],
        header_name='Log',
    )
238073
def parseruninfo(self):
        """"""
        # Check if the RunInfo.xml file is provided, otherwise, yield N/A
        try:
            runinfo = ElementTree.ElementTree(file=self.runinfo)
            # Get the run id from the
            for elem in runinfo.iter():
                for run in elem:
                    try:
                        self.runid = run.attrib['Id']
                        self.runnumber = run.attrib['Number']
                    except KeyError:
                        break
            # pull the text from flowcell and instrument values using the .iter(tag="X") function
            for elem in runinfo.iter(tag="Flowcell"):
                self.flowcell = elem.text
            for elem in runinfo.iter(tag="Instrument"):
                self.instrument = elem.text
        except IOError:
            pass
        # Extract run statistics from either GenerateRunStatistics.xml or indexingQC.txt
        self.parserunstats()