Process the IGRA2 text file for observations at site_id matching time.

        Return:
        -------
            :class: `pandas.DataFrame` containing the body data.
            :class: `pandas.DataFrame` containing the header data.
19136
def _get_data(self):
        
        # Split the list of times into begin and end dates. If only
        # one date is supplied, set both begin and end dates equal to that date.

        body, header, dates_long, dates = self._get_data_raw()

        params = self._get_fwf_params()

        df_body = pd.read_fwf(StringIO(body), **params['body'])
        df_header = pd.read_fwf(StringIO(header), **params['header'])
        df_body['date'] = dates_long

        df_body = self._clean_body_df(df_body)
        df_header = self._clean_header_df(df_header)
        df_header['date'] = dates

        return df_body, df_header
196118
def get_frame(self):
        
        metric_items = list(self.db.metrics.find({'run_name': self.model_config.run_name}).sort('epoch_idx'))
        if len(metric_items) == 0:
            return pd.DataFrame(columns=['run_name'])
        else:
            return pd.DataFrame(metric_items).drop(['_id', 'model_name'], axis=1).set_index('epoch_idx')
1321874
def get_dataframe_row(self, dataset_cases, predicted_data, pdb_data, record_id, additional_prediction_data_columns):
        ''''''

        # Ignore derived mutations if appropriate
        record = dataset_cases[record_id]
        if self.is_this_record_a_derived_mutation(record) and not self.include_derived_mutations:
            return None

        amino_acid_details, CAA, PAA, HAA = self.amino_acid_details, self.CAA, self.PAA, self.HAA
        burial_cutoff = self.burial_cutoff

        # Initialize variables. For ambiguous cases where the set of distinct values has multiple values, we default to None
        residue_charge, residue_charges = None, set()
        exposure, exposures = None, set()
        volume_change, volume_changes = None, set()
        record_wtaa, wtaas = None, set()
        record_mutaa, mutaas = None, set()
        DSSPSimpleSSType, DSSPSimpleSSTypes = None, set()
        DSSPType, DSSPTypes = None, set()
        DSSPExposure, DSSPExposures = None, set()
        scops = set()
        mutation_string = []
        num_derivative_errors = predicted_data.get('Errors', {}).get('Derivative error count', 0)
        run_time = predicted_data.get('RunTime', None)
        max_memory = predicted_data.get('MaxMemory', None)

        mutations = self.get_record_mutations(record)
        for m in mutations:

            wtaa = m['WildTypeAA']
            mutaa = m['MutantAA']
            mutation_string.append('{0} {1}{2}{3}'.format(m['Chain'], m['WildTypeAA'], m['ResidueID'], m['MutantAA']))

            # Residue types and chain
            wtaas.add(wtaa)
            mutaas.add(mutaa)
            if m.get('SCOP class'):
                scops.add(m['SCOP class'])
            DSSPSimpleSSTypes.add(m['DSSPSimpleSSType'])
            DSSPTypes.add(m['DSSPType'])
            DSSPExposures.add(m['DSSPExposure'])

            # Burial
            if m['DSSPExposure'] != None:
                if m['DSSPExposure'] > burial_cutoff:
                    exposures.add('E')
                else:
                    exposures.add('B')
            else:
                exposures.add(None)

            # Volume
            if amino_acid_details[wtaa]['van der Waals volume'] < amino_acid_details[mutaa]['van der Waals volume']:
                volume_changes.add('SL')
            elif amino_acid_details[wtaa]['van der Waals volume'] > amino_acid_details[mutaa]['van der Waals volume']:
                volume_changes.add('LS')
            elif amino_acid_details[wtaa]['van der Waals volume'] == amino_acid_details[mutaa]['van der Waals volume']:
                volume_changes.add('XX')

            # Charge
            if ((wtaa in CAA or wtaa in PAA) and (mutaa in HAA)) or ((mutaa in CAA or mutaa in PAA) and (wtaa in HAA)):
                residue_charges.add('Change')
            elif (wtaa in CAA or wtaa in PAA) and (mutaa in CAA or mutaa in PAA):
                residue_charges.add('Polar/Charged')
            elif (wtaa in HAA) and (mutaa in HAA):
                residue_charges.add('Hydrophobic/Non-polar')
            else:
                 raise colortext.Exception('Should not reach here.')

        # Create a string representing the mutations (useful for labeling rather than analysis)
        mutation_string = '; '.join(mutation_string)

        # Taking unique values, determine the residue charges of the wildtype and mutant residues, the wildtype residue exposure, and the relative change in van der Waals volume
        if len(residue_charges) == 1: residue_charge = residue_charges.pop()
        if len(exposures) == 1: exposure = exposures.pop()
        if len(volume_changes) == 1: volume_change = volume_changes.pop()

        # Taking unique values, determine the wildtype and mutant residue types
        all_residues = wtaas.union(mutaas)
        if len(wtaas) == 1: record_wtaa = wtaas.pop()
        if len(mutaas) == 1: record_mutaa = mutaas.pop()

        # Taking unique values, determine the secondary structure and residue exposures from the DSSP data in the dataset
        if len(DSSPSimpleSSTypes) == 1: DSSPSimpleSSType = DSSPSimpleSSTypes.pop()
        if len(DSSPTypes) == 1: DSSPType = DSSPTypes.pop()
        if len(DSSPExposures) == 1: DSSPExposure = DSSPExposures.pop()

        # Determine the SCOP classification from the SCOPe data in the dataset
        full_scop_classification, scop_class, scop_fold = None, None, None
        if len(scops) > 1:
            self.log('Warning: There is more than one SCOPe class for record {0}.'.format(record_id), colortext.warning)
        elif len(scops) == 1:
            full_scop_classification = scops.pop()
            scop_tokens = full_scop_classification.split('.')
            scop_class = scop_tokens[0]
            if len(scop_tokens) > 1:
                scop_fold = '.'.join(scop_tokens[0:2])

        # Partition the data by PDB resolution with bins: N/A, <1.5, 1.5-<2.0, 2.0-<2.5, >=2.5
        pdb_record = pdb_data.get(self.get_record_pdb_file_id(record).upper())
        pdb_resolution_bin = None
        pdb_resolution = pdb_record.get('Resolution')
        if pdb_resolution != None:
            if pdb_resolution < 1.5:
                pdb_resolution_bin = '<1.5'
            elif pdb_resolution < 2.0:
                pdb_resolution_bin = '1.5-2.0'
            elif pdb_resolution < 2.5:
                pdb_resolution_bin = '2.0-2.5'
            else:
                pdb_resolution_bin = '>=2.5'
        pdb_resolution_bin = pdb_resolution_bin or 'N/A'

        # Mark mutations involving glycine or proline
        has_gp_mutation = 'G' in all_residues or 'P' in all_residues

        # Create the data matrix
        dataframe_record = dict(
            DatasetID = record_id,
            PDBFileID = self.get_record_pdb_file_id(record),
            Mutations = mutation_string,
            NumberOfMutations = len(mutations),
            Predicted = predicted_data[self.ddg_analysis_type],
            ResidueCharges = residue_charge,
            VolumeChange = volume_change,
            HasGPMutation = int(has_gp_mutation),
            WildTypeDSSPType = DSSPType,
            WildTypeDSSPSimpleSSType = DSSPSimpleSSType,
            WildTypeDSSPExposure = DSSPExposure,
            WildTypeSCOPClass = scop_class,
            WildTypeSCOPFold = scop_fold,
            WildTypeSCOPClassification = full_scop_classification,
            WildTypeExposure = exposure,
            WildTypeAA = record_wtaa,
            MutantAA = record_mutaa,
            PDBResolution = pdb_record.get('Resolution'),
            PDBResolutionBin = pdb_resolution_bin,
            NumberOfResidues = self.count_residues(record, pdb_record) or None,
            NumberOfDerivativeErrors = num_derivative_errors,
            RunTime = run_time,
            MaxMemory = max_memory,
            )
        for c in additional_prediction_data_columns:
            dataframe_record[c] = predicted_data.get(c)

        if self.contains_experimental_data:
            # These fields are particular to dataframes containing experimental values e.g. for benchmarking runs or for
            # datasets where we have associated experimental values
            self.get_experimental_ddg_values(record, dataframe_record)
            self.compute_stability_classification(predicted_data, record, dataframe_record)
            self.compute_absolute_error(predicted_data, record, dataframe_record)

        return dataframe_record
1198663
def get_experimental_ddg_values(self, record, dataframe_record):
        ''''''
        new_idxs = []
        for analysis_set in self.get_analysis_sets(record):
            ddg_details = record['DDG'][analysis_set]
            exp_ddg_fieldname = BenchmarkRun.get_analysis_set_fieldname('Experimental', analysis_set)
            new_idxs.append(exp_ddg_fieldname)
            dataframe_record[exp_ddg_fieldname] = None
            if ddg_details:
                dataframe_record[exp_ddg_fieldname] = ddg_details['MeanDDG']

        # Update the CSV headers
        try:
            idx = self.csv_headers.index('Experimental')
            self.csv_headers = self.csv_headers[:idx] + new_idxs + self.csv_headers[idx + 1:]
        except ValueError, e: pass
38335
def as_dataframe(self, absolute_time=False):
        

        import pandas as pd

        # When absolute_time is True,
        # use the wf_start_time as offset for the time_track()
        try:
            time = self.time_track(absolute_time)
        except KeyError:
            time = None
        if self.channel is None:
            return pd.DataFrame.from_items(
                [(ch.channel, pd.Series(ch.data))
                    for ch in self.tdms_file.group_channels(self.group)])
        else:
            return pd.DataFrame(self._data, index=time, columns=[self.path])
530989
def parse_report(self):
        
        logging.info('Parsing Plasmid Extractor outputs')
        # A dictionary to store the parsed excel file in a more readable format
        nesteddictionary = dict()
        # Use pandas to read in the CSV file, and convert the pandas data frame to a dictionary (.to_dict())
        dictionary = pandas.read_csv(self.plasmid_report).to_dict()
        # Iterate through the dictionary - each header from the CSV file
        for header in dictionary:
            # Sample is the primary key, and value is the value of the cell for that primary key + header combination
            for sample, value in dictionary[header].items():
                # Update the dictionary with the new data
                try:
                    nesteddictionary[sample].update({header: value})
                # Create the nested dictionary if it hasn't been created yet
                except KeyError:
                    nesteddictionary[sample] = dict()
                    nesteddictionary[sample].update({header: value})
        # Get the results into the metadata object
        for sample in self.metadata:
            # Initialise the plasmid extractor genobject
            setattr(sample, self.analysistype, GenObject())
            # Initialise the list of all plasmids
            sample[self.analysistype].plasmids = list()
            # Iterate through the dictionary of results
            for line in nesteddictionary:
                # Extract the sample name from the dictionary in a manner consistent with the rest of the COWBAT
                # pipeline e.g. 2014-SEQ-0276_S2_L001 becomes 2014-SEQ-0276
                sample_name = nesteddictionary[line]['Sample']
                # Use the filer method to extract the name
                name = list(filer([sample_name]))[0]
                # Ensure that the names match
                if name == sample.name:
                    # Append the plasmid name extracted from the dictionary to the list of plasmids
                    sample[self.analysistype].plasmids.append(nesteddictionary[line]['Plasmid'])
        # Copy the report to the folder containing all reports for the pipeline
        try:
            shutil.copyfile(self.plasmid_report, os.path.join(self.reportpath, 'plasmidReport.csv'))
        except IOError:
            pass
66510
def get_ped_info(data, samples):
    
    family_id = tz.get_in(["metadata", "family_id"], data, None)
    if not family_id:
        family_id = _find_shared_batch(samples)
    return {
        "gender": {"male": 1, "female": 2, "unknown": 0}.get(get_gender(data)),
        "individual_id": dd.get_sample_name(data),
        "family_id": family_id,
        "maternal_id": tz.get_in(["metadata", "maternal_id"], data, -9),
        "paternal_id": tz.get_in(["metadata", "paternal_id"], data, -9),
        "affected": get_affected_status(data),
        "ethnicity": tz.get_in(["metadata", "ethnicity"], data, -9)
    }
475631
def get_dataframe(self, sort_key="wall_time", **kwargs):
        
        import pandas as pd
        frame = pd.DataFrame(columns=AbinitTimerSection.FIELDS)

        for osect in self.order_sections(sort_key):
            frame = frame.append(osect.to_dict(), ignore_index=True)

        # Monkey patch
        frame.info =  self.info
        frame.cpu_time =  self.cpu_time
        frame.wall_time =  self.wall_time
        frame.mpi_nprocs =  self.mpi_nprocs
        frame.omp_nthreads =  self.omp_nthreads
        frame.mpi_rank =  self.mpi_rank
        frame.fname =  self.fname

        return frame
633863
def _process(self, data):
        
        indices = []
        for indexname, indexdata in list(data.items()):
            idata = []
            for docname, docdata in list(indexdata.get("mappings", {}).items()):
                o = get_field(docname, docdata, document_object_field=self.document_object_field, is_document=True)
                o.connection = self.connection
                o.index_name = indexname
                idata.append((docname, o))
            idata.sort()
            indices.append((indexname, idata))
        indices.sort()
        self.indices = OrderedDict(indices)
84969
def get_header_values(self):
        
        lines = self.getOriginalFile().data.splitlines()
        reader = csv.reader(lines)
        header_fields = header_data = []
        for row in reader:
            if not any(row):
                continue
            if row[0].strip().lower() == 'header':
                header_fields = [x.strip() for x in row][1:]
                continue
            if row[0].strip().lower() == 'header data':
                header_data = [x.strip() for x in row][1:]
                break
        if not (header_data or header_fields):
            return None
        if not (header_data and header_fields):
            self.error("File is missing header row or header data")
            return None
        # inject us out of here
        values = dict(zip(header_fields, header_data))
        # blank cell from sheet will probably make it in here:
        if '' in values:
            del (values[''])
        return values
591651
def to_timestamp(self, data):
        
        result = pd.Series(index=data.index)
        _slice = ~data[self.col_name].isnull()

        result[_slice] = data[_slice][self.col_name].astype('int64')
        return result
340776
def get_data_statistics(interpreted_files):
    '''
    '''
    print '| *File Name* | *File Size* | *Times Stamp* | *Events* | *Bad Events* | *Measurement time* | *# SR* | *Hits* |'  # Mean Tot | Mean rel. BCID'
    for interpreted_file in interpreted_files:
        with tb.open_file(interpreted_file, mode="r") as in_file_h5:  # open the actual hit file
            n_hits = np.sum(in_file_h5.root.HistOcc[:])
            measurement_time = int(in_file_h5.root.meta_data[-1]['timestamp_stop'] - in_file_h5.root.meta_data[0]['timestamp_start'])
#             mean_tot = np.average(in_file_h5.root.HistTot[:], weights=range(0,16) * np.sum(range(0,16)))# / in_file_h5.root.HistTot[:].shape[0]
#             mean_bcid = np.average(in_file_h5.root.HistRelBcid[:], weights=range(0,16))
            n_sr = np.sum(in_file_h5.root.HistServiceRecord[:])
            n_bad_events = int(np.sum(in_file_h5.root.HistErrorCounter[2:]))
            try:
                n_events = str(in_file_h5.root.Hits[-1]['event_number'] + 1)
            except tb.NoSuchNodeError:
                n_events = '~' + str(in_file_h5.root.meta_data[-1]['event_number'] + (in_file_h5.root.meta_data[-1]['event_number'] - in_file_h5.root.meta_data[-2]['event_number']))
            else:
                print '|', os.path.basename(interpreted_file), '|', int(os.path.getsize(interpreted_file) / (1024.0 * 1024.0)), 'Mb |', time.ctime(os.path.getctime(interpreted_file)), '|', n_events, '|', n_bad_events, '|', measurement_time, 's |', n_sr, '|', n_hits, '|'
403610
def to_dataframe(self):
        """"""
        # list of properties to extract from Variant objects if they're
        # not None
        variant_properties = [
            "contig",
            "start",
            "ref",
            "alt",
            "is_snv",
            "is_transversion",
            "is_transition"
        ]

        def row_from_effect(effect):
            row = OrderedDict()

            row['variant'] = str(effect.variant.short_description)
            for field_name in variant_properties:
                # if effect.variant is None then this column value will be None
                row[field_name] = getattr(effect.variant, field_name, None)
            row['gene_id'] = effect.gene_id
            row['gene_name'] = effect.gene_name
            row['transcript_id'] = effect.transcript_id
            row['transcript_name'] = effect.transcript_name

            row['effect_type'] = effect.__class__.__name__
            row['effect'] = effect.short_description
            return row
        return pd.DataFrame.from_records([row_from_effect(effect) for effect in self])
198857
def get_df(self, data_file):
        

        crawler = QAHistoryFinancialCrawler()

        with open(data_file, 'rb') as df:
            data = crawler.parse(download_file=df)

        return crawler.to_df(data)
149089
def _get_dataframe(self):
        
        if self.source_conn is None:  # Use local file
            return ge.read_csv(self.dataset_name, **self.dataset_params)

        if isinstance(self.source_conn, S3Hook):
            hook = ExpectationS3CsvHook(aws_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, self.source_bucket_name, **self.dataset_params)

        if isinstance(self.source_conn, DbApiHook):
            hook = ExpectationMySQLHook(mysql_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, **self.dataset_params)
623904
def process_icmp_frame(self,
                           id=None,
                           msg=None):
        

        # normalize into a dataframe
        df = json_normalize(msg)
        # convert to a flattened dictionary
        dt = json.loads(df.to_json())

        flat_msg = {}

        for k in dt:
            new_key = "icmp_{}".format(k)
            flat_msg[new_key] = dt[k]["0"]
            if new_key not in self.icmp_keys:
                self.icmp_keys[new_key] = k
        # end of capturing all unique keys

        dt["icmp_id"] = id
        self.all_icmp.append(dt)

        log.debug("ICMP data updated:")
        log.debug(self.icmp_keys)
        log.debug(self.all_icmp)
        log.debug("")

        return flat_msg
1281795
def _process_data(
        self,
        data,
        collect_aggregates_only,
        process_events,
        url=None,
        collect_status_metrics=False,
        collect_status_metrics_by_host=False,
        tag_service_check_by_host=False,
        services_incl_filter=None,
        services_excl_filter=None,
        collate_status_tags_per_host=False,
        count_status_by_service=True,
        custom_tags=None,
        tags_regex=None,
        active_tag=None,
        enable_service_check=False,
    ):
        '''  '''

        # Split the first line into an index of fields
        # The line looks like (broken up onto multiple lines)
        # "# pxname,svname,qcur,qmax,scur,smax,slim,
        # stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,
        # wredis,status,weight,act,bck,chkfail,chkdown,lastchg,
        # downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,
        # type,rate,rate_lim,rate_max,"
        fields = []
        for f in data[0].split(','):
            if f:
                f = f.replace('# ', '')
                fields.append(f.strip())

        self.hosts_statuses = defaultdict(int)

        back_or_front = None

        # Sanitize CSV, handle line breaks
        data = self._sanitize_lines(data)
        custom_tags = [] if custom_tags is None else custom_tags
        active_tag = [] if active_tag is None else active_tag

        # First initialize here so that it is defined whether or not we enter the for loop
        line_tags = list(custom_tags)

        # Skip the first line, go backwards to set back_or_front
        for line in data[:0:-1]:
            if not line.strip():
                continue

            # Store each line's values in a dictionary
            data_dict = self._line_to_dict(fields, line)

            if self._is_aggregate(data_dict):
                back_or_front = data_dict['svname']

            self._update_data_dict(data_dict, back_or_front)

            self._update_hosts_statuses_if_needed(
                collect_status_metrics, collect_status_metrics_by_host, data_dict, self.hosts_statuses
            )

            # Clone the list to avoid extending the original
            # which would carry over previous iteration tags
            line_tags = list(custom_tags)

            regex_tags = self._tag_from_regex(tags_regex, data_dict['pxname'])
            if regex_tags:
                line_tags.extend(regex_tags)

            if self._should_process(data_dict, collect_aggregates_only):
                # update status
                # Send the list of data to the metric and event callbacks
                self._process_metrics(
                    data_dict,
                    url,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                    active_tag=active_tag,
                )
            if process_events:
                self._process_event(
                    data_dict,
                    url,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                )
            if enable_service_check:
                self._process_service_check(
                    data_dict,
                    url,
                    tag_by_host=tag_service_check_by_host,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                )

        if collect_status_metrics:
            self._process_status_metric(
                self.hosts_statuses,
                collect_status_metrics_by_host,
                services_incl_filter=services_incl_filter,
                services_excl_filter=services_excl_filter,
                collate_status_tags_per_host=collate_status_tags_per_host,
                count_status_by_service=count_status_by_service,
                custom_tags=line_tags,
                active_tag=active_tag,
            )

            self._process_backend_hosts_metric(
                self.hosts_statuses,
                services_incl_filter=services_incl_filter,
                services_excl_filter=services_excl_filter,
                custom_tags=line_tags,
                active_tag=active_tag,
            )

        return data