Read er_ages.txt file.
        Parse information into dictionaries for each site/sample.
        Then add it to the site/sample object as site/sample.age_data.
163339
def get_age_info(self, filename=None):
        
        # use filename if provided, otherwise find er_ages.txt in WD
        if not filename:
            short_filename = 'er_ages.txt'
            magic_file = os.path.join(self.WD, short_filename)
        else:
            magic_file = filename
        if not os.path.isfile(magic_file):
            print('-W- Could not find {}'.format(magic_file))
            return False

        data_dict, header, file_type = self.read_magic_file(magic_file, 'by_line_number')
        # if provided file is not an age_file,
        # try to read it in as whatever type of file it actually is
        if file_type != 'er_ages':
            item_type = file_type.split('_')[1][:-1]
            self.get_magic_info(item_type, filename=filename, sort_by_file_type=True)
            return file_type

        # if it is an age file,
        # determine level for each age and assign it to the appropriate pmag object
        for item_dict in list(data_dict.values()):
            item_type = None
            for dtype in ['specimen', 'sample', 'site', 'location']:
                header_name = 'er_' + dtype + '_name'
                if header_name in list(item_dict.keys()):
                    if item_dict[header_name]:
                        item_type = dtype
                        item_name = item_dict[header_name].strip()
                        break
            if not item_type:
                print('-W- You must provide a name for your age')
                print('    These data:\n{}\n    will not be imported'.format(item_dict))
                continue
            items_list = self.data_lists[item_type][0]
            item = self.find_by_name(item_name, items_list)
            if not item:
                ## the following code creates any item in er_ages that does not exist already
                ## however, we may not WANT that behavior
                print("""-I- A {} named {} in your age file was not found in the data object:
    Now initializing {} {}""".format(item_type, item_name, item_type, item_name))
                ind = self.ancestry.index(item_type)
                parent_type = self.ancestry[ind+1]
                parent_header, parent_constructor = None, None
                if parent_type:
                    parent_list, parent_class, parent_constructor = self.data_lists[parent_type]
                    parent_header = 'er_' + parent_type + '_name'
                parent_name = item_dict.get(parent_header, '')
                parent = self.find_by_name(parent_name, parent_list)
                # if the parent item doesn't exist, and should, create it
                if parent_name and not parent:
                    print("""-I- A {} named {} in your age file was not found in the data object:
    Now initializing {} {}""".format(parent_type, parent_name, parent_type, parent_name))
                    parent = parent_constructor(parent_name, None)
                item_constructor = self.data_lists[item_type][2]
                if not parent:
                    parent_name = None
                item = item_constructor(item_name, parent_name)
            # add the age data to the object
            item.age_data = remove_dict_headers(item_dict)
        # note that data is available to write
        self.write_ages = True
        return file_type
1272820
def read
      @site.layouts = LayoutReader.new(site).read
      read_directories
      read_included_excludes
      sort_files!
      @site.data = DataReader.new(site).read(site.config["data_dir"])
      CollectionReader.new(site).read
      ThemeAssetsReader.new(site).read
    end
1108459
def readBerSANS(filename):
    """"""
    hed = {'Comment': ''}
    translate = {'Lambda': 'Wavelength',
                 'Title': 'Owner',
                 'SampleName': 'Title',
                 'BeamcenterX': 'BeamPosY',
                 'BeamcenterY': 'BeamPosX',
                 'Time': 'MeasTime',
                 'TotalTime': 'MeasTime',
                 'Moni1': 'Monitor',
                 'Moni2': 'Monitor',
                 'Moni': 'Monitor',
                 'Transmission': 'Transm',
                 }
    with open(filename, 'rt') as f:
        comment_next = False
        for l in f:
            l = l.strip()
            if comment_next:
                hed['Comment'] = hed['Comment'] + '\n' + l
                comment_next = False
            elif l.startswith('%Counts'):
                break
            elif l.startswith('%Comment'):
                comment_next = True
            elif l.startswith('%'):
                continue
            elif l.split('=', 1)[0] in translate:
                hed[translate[l.split('=', 1)[0]]] = misc.parse_number(
                    l.split('=', 1)[1])
            else:
                try:
                    hed[l.split('=', 1)[0]] = misc.parse_number(
                        l.split('=', 1)[1])
                except IndexError:
                    print(l.split('=', 1))
    if 'FileName' in hed:
        m = re.match('D(\d+)\.(\d+)', hed['FileName'])
        if m is not None:
            hed['FSN'] = int(m.groups()[0])
            hed['suffix'] = int(m.groups()[1])
    if 'FileDate' in hed:
        hed['Date'] = dateutil.parser.parse(hed['FileDate'])
    if 'FileTime' in hed:
        hed['Date'] = datetime.datetime.combine(
            hed['Date'].date(), dateutil.parser.parse(hed['FileTime']).time())
    hed['__Origin__'] = 'BerSANS'
    if 'SD' in hed:
        hed['Dist'] = hed['SD'] * 1000
    if hed['Comment'].startswith('\n'):
        hed['Comment'] = hed['Comment'][1:]
    hed['__particle__'] = 'neutron'
    hed['Wavelength'] *= 10  # convert from nanometres to Angstroems
    return hed
1224988
def _get_data_raw(self):
        
        # Import need to be here so we can monkeypatch urlopen for testing and avoid
        # downloading live data for testing
        try:
            from urllib.request import urlopen
        except ImportError:
            from urllib2 import urlopen

        with closing(urlopen(self.ftpsite + self.site_id + self.suffix + '.zip')) as url:
            f = ZipFile(BytesIO(url.read()), 'r').open(self.site_id + self.suffix)

        lines = [line.decode('utf-8') for line in f.readlines()]

        body, header, dates_long, dates = self._select_date_range(lines)

        return body, header, dates_long, dates
844326
def get_sample_data(sample_file):
    

    sequence_sample_in_fasta = None
    with open(sample_file) as handle:
        sequence_sample_in_fasta = handle.read()

    return sequence_sample_in_fasta
872696
def parse_into_object(self, res, text):
        """"""

        text = tounicode(text, encoding="utf-8")

        m = self.start_dict_re.match(text, 0)
        if m:
            i = self._parse_dict_into_object(res, text, 1)
        else:
            self._fail("not correct file format", text, 0)
        if text[i:].strip():
            self._fail("Unexpected trailing content", text, i)
        return i
1351232
def parse_readme():
    """"""
    # Get the long description from the relevant file
    here = os.path.abspath(os.path.dirname(__file__))
    readme_path = os.path.join(here, 'README.md')
    with codecs.open(readme_path, encoding='utf-8') as handle:
        long_description = handle.read()

    return long_description
908808
def read
      site.collections.each_value do |collection|
        collection.read unless SPECIAL_COLLECTIONS.include?(collection.label)
      end
    end
134737
def _read_content(self, response: Response, original_url_info: URLInfo):
        ''''''
        data = response.body.read(4096)
        url_info = original_url_info

        try:
            self._robots_txt_pool.load_robots_txt(url_info, data)
        except ValueError:
            _logger.warning(__(
                _('Failed to parse {url} for robots exclusion rules. '
                  'Ignoring.'), url_info.url))
            self._accept_as_blank(url_info)
        else:
            _logger.debug(__('Got a good robots.txt for {0}.',
                             url_info.url))
136930
def parse_sampleinfo(data: dict) -> dict:
    

    genome_build = data['human_genome_build']
    genome_build_str = f"{genome_build['source']}{genome_build['version']}"
    if 'svdb' in data['program']:
        svdb_outpath = (f"{data['program']['svdb']['path']}")
    else:
        svdb_outpath = ''
    outdata = {
        'date': data['analysis_date'],
        'family': data['family'],
        'genome_build': genome_build_str,
        'rank_model_version': data['program']['genmod']['rank_model']['version'],
        'is_finished': True if data['analysisrunstatus'] == 'finished' else False,
        'pedigree_path': data['pedigree_minimal'],
        'peddy': {
            'ped': (data['program']['peddy']['peddy']['path'] if
                    'peddy' in data['program'] else None),
            'ped_check': (data['program']['peddy']['ped_check']['path'] if
                          'peddy' in data['program'] else None),
            'sex_check': (data['program']['peddy']['sex_check']['path'] if
                          'peddy' in data['program'] else None),
        },
        'qcmetrics_path': data['program']['qccollect']['path'],
        'samples': [],
        'snv': {
            'bcf': data['most_complete_bcf']['path'],
            'clinical_vcf': data['vcf_binary_file']['clinical']['path'],
            'gbcf': data['gbcf_file']['path'],
            'research_vcf': data['vcf_binary_file']['research']['path'],
        },
        'svdb_outpath': svdb_outpath,
        'sv': {
            'bcf': data.get('sv_bcf_file', {}).get('path'),
            'clinical_vcf': (data['sv_vcf_binary_file']['clinical']['path'] if
                             'sv_vcf_binary_file' in data else None),
            'merged': svdb_outpath,
            'research_vcf': (data['sv_vcf_binary_file']['research']['path'] if
                             'sv_vcf_binary_file' in data else None),
        },
        'version': data['mip_version'],
    }

    for sample_id, sample_data in data['sample'].items():
        sample = {
            'id': sample_id,
            'bam': sample_data['most_complete_bam']['path'],
            'sambamba': list(sample_data['program']['sambamba_depth'].values())[0]['path'],
            'sex': sample_data['sex'],
            # subsample mt is only for wgs data
            'subsample_mt': (list(sample_data['program']['samtools_subsample_mt'].values())[0]['path'] if
                             'samtools_subsample_mt' in sample_data['program'] else None),
            'vcf2cytosure': list(sample_data['program']['vcf2cytosure'].values())[0]['path'],
        }
        chanjo_sexcheck = list(sample_data['program']['chanjo_sexcheck'].values())[0]
        sample['chanjo_sexcheck'] = chanjo_sexcheck['path']
        outdata['samples'].append(sample)

    return outdata
708733
def create_read_parser():
    import argparse
    parent_read_parser = argparse.ArgumentParser(add_help=False)
    parent_read_parser.add_argument(
        'input_file',
        help=('Path to file containing input ids of content to read. For the '
              'no-db options, this is simply a file with each line being a '
              'pmid. For the with-db options, this is a file where each line '
              'is of the form \'<id type>:<id>\', for example \'pmid:12345\'')
    )
    parent_read_parser.add_argument(
        '--start_ix',
        type=int,
        help='Start index of ids to read.'
    )
    parent_read_parser.add_argument(
        '--end_ix',
        type=int,
        help='End index of ids to read. If `None`, read content from all ids.'
    )
    parent_read_parser.add_argument(
        '--force_read',
        action='store_true',
        help='Read papers even if previously read by current REACH.'
    )
    parent_read_parser.add_argument(
        '--force_fulltext',
        action='store_true',
        help='Get full text content even if content already on S3.'
    )
    parent_read_parser.add_argument(
        '--ids_per_job',
        default=3000,
        type=int,
        help='Number of PMIDs to read for each AWS Batch job.'
    )
    ''' 
    '''
    return parent_read_parser
1023486
def _prepare_sample(data, run_folder):
    
    want = set(["description", "files", "genome_build", "name", "analysis", "upload", "algorithm"])
    out = {}
    for k, v in data.items():
        if k in want:
            out[k] = _relative_paths(v, run_folder)
    if "algorithm" not in out:
        analysis, algorithm = _select_default_algorithm(out.get("analysis"))
        out["algorithm"] = algorithm
        out["analysis"] = analysis
    description = "%s-%s" % (out["name"], clean_name(out["description"]))
    out["name"] = [out["name"], description]
    out["description"] = description
    return out
921805
def _get_age(columns):
    

    # Need to check multiple places for age, year, or yrbp
    # 1. Check the column variable name
    # 2. Check for an "inferredVariableType

    age = []
    try:
        # Step 1:
        # Check for age first (exact match)
        if "age" in columns:
            # Save the values
            age = columns["age"]["values"]
        # Check for year second (exact match)
        elif "year" in columns:
            # Save the values
            age = columns["year"]["values"]
        elif "yrbp" in columns:
            # Save the values
            age = columns["yrbp"]["values"]

        # Step 2 No exact matches, check for an "inferredVariableType" : "Age" or "Year"
        if not age:
            # Loop through column variableNames
            for k, v in columns.items():
                try:
                    if v["inferredVariableType"].lower() == "age":
                        age = v["values"]
                    elif v["inferredVariableType"].lower() == "year":
                        age = v["values"]
                except Exception:
                    # Not too concerned if we error here.
                    pass

        # Step 3: No year or age found, start searching for a loose match with "age" or "year" in the variableName.
        if not age:
            # Loop through column variableNames
            for k, v in columns.items():
                k_low = k.lower()
                # Check for age in the variableName (loose match)
                if "age" in k_low:
                    # Save the values
                    age = v["values"]
                # Check for year in variableName (loose match)
                elif "year" in k_low:
                    # Save the values
                    age = v["values"]
                elif "yrbp" in k_low:
                    # Save the values
                    age = v["values"]

    # If we expected a dictionary, and didn't get one
    except AttributeError as e:
        logger_inferred_data.warn("get_age: AttributeError: {}".format(e))
    # If we were looking for values, and didn't get one
    except KeyError as e:
        logger_inferred_data.warn("get_age: KeyError: {}".format(e))
    # Fail-safe for other problems
    except Exception as e:
        logger_inferred_data.warn("get_age: Exception: {}".format(e))

    return age
334357
def parse_ouput(self, data, context, response):
        '''
        
        '''

        parsedData = data.split("!-!")
        nameList = ['user_sessions', 'group_membership.csv', 'acls.csv', 'local_admins.csv', 'trusts.csv']
        for x in range(0, len(parsedData)):
            if "ComputerName" in parsedData[x] and "UserName" in parsedData[x] :
                log_name = '{}-{}-{}.csv'.format(nameList[0], response.client_address[0], datetime.now().strftime("%Y-%m-%d_%H%M%S"))
                write_log(parsedData[x].replace('" "', '"\n"').replace(' "', '"'), log_name)
                context.log.info("Saved csv output to {}".format(log_name))
            elif "GroupName" in parsedData[x] and "AccountName" in parsedData[x] :
                log_name = '{}-{}-{}.csv'.format(nameList[1], response.client_address[0], datetime.now().strftime("%Y-%m-%d_%H%M%S"))
                write_log(parsedData[x].replace('" "', '"\n"').replace(' "', '"'), log_name)
                context.log.info("Saved csv output to {}".format(log_name))
            elif "ComputerName" in parsedData[x] and "AccountName" in parsedData[x] :
                log_name = '{}-{}-{}.csv'.format(nameList[3], response.client_address[0], datetime.now().strftime("%Y-%m-%d_%H%M%S"))
                write_log(parsedData[x].replace('" "', '"\n"').replace(' "', '"'), log_name)
                context.log.info("Saved csv output to {}".format(log_name))
            elif "SourceDomain" in parsedData[x] and "TrustType" in parsedData[x] :
                log_name = '{}-{}-{}.csv'.format(nameList[4], response.client_address[0], datetime.now().strftime("%Y-%m-%d_%H%M%S"))
                write_log(parsedData[x].replace('" "', '"\n"').replace(' "', '"'), log_name)
                context.log.info("Saved csv output to {}".format(log_name))
            elif "ObjectName" in parsedData[x] and "ObjectType" in parsedData[x] :
                log_name = '{}-{}-{}.csv'.format(nameList[2], response.client_address[0], datetime.now().strftime("%Y-%m-%d_%H%M%S"))
                write_log(parsedData[x].replace('" "', '"\n"').replace(' "', '"'), log_name)
                context.log.info("Saved csv output to {}".format(log_name))
878115
def sample_reads(self):
        
        logging.info('Read sampling')
        for sample in self.metadata:
            # Iterate through all the desired depths of coverage
            for depth in self.read_depths:
                for read_pair in self.read_lengths:
                    # Set the name of the output directory
                    sample.sampled_reads[depth][read_pair].sampled_outputdir \
                        = os.path.join(sample.sampled_reads[depth][read_pair].outputdir, 'sampled')
                    # Set the name of the forward reads - include the depth and read length information
                    sample.sampled_reads[depth][read_pair].forward_reads.fastq = \
                        os.path.join(sample.sampled_reads[depth][read_pair].sampled_outputdir,
                                     '{name}_{depth}_{read_pair}_R1.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    # Reverse reads
                    sample.sampled_reads[depth][read_pair].reverse_reads.fastq = \
                        os.path.join(sample.sampled_reads[depth][read_pair].sampled_outputdir,
                                     '{name}_{depth}_{read_pair}_R2.fastq.gz'
                                     .format(name=sample.name,
                                             depth=depth,
                                             read_pair=read_pair))
                    logging.info(
                        'Sampling {num_reads} paired reads for sample {name} with the following parameters:\n'
                        'depth {dp}, forward reads {fl}bp, and reverse reads {rl}bp'
                        .format(num_reads=sample.simulated_reads[depth][read_pair].num_reads,
                                dp=depth,
                                name=sample.name,
                                fl=sample.sampled_reads[depth][read_pair].forward_reads.length,
                                rl=sample.sampled_reads[depth][read_pair].reverse_reads.length))
                    # Use the reformat method in the OLCTools bbtools wrapper
                    # Note that upsample=t is used to ensure that the target number of reads (samplereadstarget) is met
                    if not os.path.isfile(sample.sampled_reads[depth][read_pair].forward_reads.trimmed_sampled_fastq):
                        out, \
                            err, \
                            sample.sampled_reads[depth][read_pair].sample_call = bbtools \
                            .reformat_reads(forward_in=sample.sampled_reads[depth][read_pair].trimmed_forwardfastq,
                                            reverse_in=sample.sampled_reads[depth][read_pair].trimmed_reversefastq,
                                            forward_out=sample.sampled_reads[depth][read_pair].forward_reads.fastq,
                                            reverse_out=sample.sampled_reads[depth][read_pair].reverse_reads.fastq,
                                            returncmd=True,
                                            **{'samplereadstarget': sample.simulated_reads[depth][read_pair].num_reads,
                                               'upsample': 't',
                                               'minlength':
                                                   sample.sampled_reads[depth][read_pair].forward_reads.length,
                                               'ziplevel': '9',
                                               'tossbrokenreads': 't',
                                               'tossjunk': 't',
                                               'Xmx': self.mem
                                               }
                                            )
                    # # Remove the trimmed reads, as they are no longer necessary
                    # try:
                    #     os.remove(sample.sampled_reads[depth][read_pair].trimmed_forwardfastq)
                    #     os.remove(sample.sampled_reads[depth][read_pair].trimmed_reversefastq)
                    # except FileNotFoundError:
                    #     pass
            # Update the JSON file
            self.write_json(sample)
999354
def parse(self, line):
        """"""
        csv_list = line.split(",")
        date_time_message = csv_list.pop(0).split(" ", 2)
        otherinfo = dict()

        for item in csv_list:
            key_value_pair = item.split(":", 1)
            key = key_value_pair[0].strip()

            if len(key_value_pair) > 1:
                value = key_value_pair[1].strip()
                if not value:
                    value = "-"
            else:
                value = "-"

            otherinfo[key] = value

        self.message = '%s\n' \
                       'Date: %s\n' \
                       'Time: %s\n' \
                       'Request: %s\n' \
                       'Referrer: %s\n' \
                       'Server: %s\n' \
                       'Client: %s\n' \
                       'Host: %s\n' \
                       'Upstream: %s\n'
        self.params = [
            date_time_message[2],
            date_time_message[0],
            date_time_message[1],
            otherinfo.get("request", "-"),
            otherinfo.get("referrer", "-"),
            otherinfo.get("server", "-"),
            otherinfo.get("client", "-"),
            otherinfo.get("host", "-"),
            otherinfo.get("upstream", "-"),
        ]
        self.site = otherinfo.get("referrer", "-")
689401
def parse(text, base=None, chop=False):
    '''
    
    '''
    #TODO: use base to facilitate relative time offsets
    text = text.strip().lower()
    text = (''.join([c for c in text if c.isdigit()]))

    if chop:
        text = text[:17]

    # TODO: support relative time offsets here...

    tlen = len(text)
    if tlen == 4:
        dt = datetime.datetime.strptime(text, '%Y')

    elif tlen == 6:
        dt = datetime.datetime.strptime(text, '%Y%m')

    elif tlen == 8:
        dt = datetime.datetime.strptime(text, '%Y%m%d')

    elif tlen == 10:
        dt = datetime.datetime.strptime(text, '%Y%m%d%H')

    elif tlen == 12:
        dt = datetime.datetime.strptime(text, '%Y%m%d%H%M')

    elif tlen == 14:
        dt = datetime.datetime.strptime(text, '%Y%m%d%H%M%S')

    elif tlen in (15, 16, 17):
        dt = datetime.datetime.strptime(text, '%Y%m%d%H%M%S%f')

    else:
        raise s_exc.BadTypeValu(valu=text, name='time',
                                mesg='Unknown time format')

    epoch = datetime.datetime(1970, 1, 1)
    return int((dt - epoch).total_seconds() * 1000)