Search all dicoms in the dicom directory, sort and validate them

    fast_read = True will only read the headers not the data
14719
def _get_grouped_dicoms(dicom_input):
    
    # if all dicoms have an instance number try sorting by instance number else by position
    if [d for d in dicom_input if 'InstanceNumber' in d]:
        dicoms = sorted(dicom_input, key=lambda x: x.InstanceNumber)
    else:
        dicoms = common.sort_dicoms(dicom_input)
    # now group per stack
    grouped_dicoms = [[]]  # list with first element a list
    timepoint_index = 0
    previous_stack_position = -1

    # loop over all sorted dicoms
    stack_position_tag = Tag(0x2001, 0x100a)  # put this there as this is a slow step and used a lot
    for index in range(0, len(dicoms)):
        dicom_ = dicoms[index]
        stack_position = 0
        if stack_position_tag in dicom_:
            stack_position = common.get_is_value(dicom_[stack_position_tag])
        if previous_stack_position == stack_position:
            # if the stack number is the same we move to the next timepoint
            timepoint_index += 1
            if len(grouped_dicoms) <= timepoint_index:
                grouped_dicoms.append([])
        else:
            # if it changes move back to the first timepoint
            timepoint_index = 0
        grouped_dicoms[timepoint_index].append(dicom_)
        previous_stack_position = stack_position

    return grouped_dicoms
44161
def _read_header(self):
        '''
        
        '''
        self._fh.seek(0)
        buf = self._fh.read(4*2)
        fc, dc = struct.unpack("<II", buf)
        return fc, dc
208013
def _read_header(self):
        
        if not self.filename or self.header_types:
            return
        rows = csv.reader(open(self.filename))
        #header = rows.next()
        header = next(rows)
        self.header_types = {} # {attr_name:type}
        self._class_attr_name = None
        self.header_order = [] # [attr_name,...]
        for el in header:
            matches = ATTR_HEADER_PATTERN.findall(el)
            assert matches, "Invalid header element: %s" % (el,)
            el_name, el_type, el_mode = matches[0]
            el_name = el_name.strip()
            self.header_order.append(el_name)
            self.header_types[el_name] = el_type
            if el_mode == ATTR_MODE_CLASS:
                assert self._class_attr_name is None, \
                    "Multiple class attributes are not supported."
                self._class_attr_name = el_name
            else:
                assert self.header_types[el_name] != ATTR_TYPE_CONTINUOUS, \
                    "Non-class continuous attributes are not supported."
        assert self._class_attr_name, "A class attribute must be specified."
721649
def check_file_type(files):
   
   all_are_fasta = True
   all_are_reads = True
   all_are_empty = True
   if sys.version_info < (3, 0):
      if isinstance(files, (str, unicode)): files = [files]
   else:
      if isinstance(files, str): files = [files]
   for file_ in files:
      debug.log('Checking file type: %s'%file_)
      # Check if file is empty
      if os.stat(file_).st_size == 0: continue
      else: all_are_empty = False
      with open_(file_) as f:
         fc = f.readline()[0]
         if fc != "@": all_are_reads = False
         if fc != ">": all_are_fasta = False
   if all_are_empty:   return 'empty'
   elif all_are_fasta: return 'fasta'
   elif all_are_reads: return 'fastq'
   else: return 'other'
634586
def _read_header(fid):
    
    fid.seek(0, SEEK_SET)
    fFileSignature = fid.read(4)
    assert fFileSignature == b'ABF2', 'only format ABF2 is currently supported'

    header = {}
    for key, offset, fmt in headerDescriptionV2:
        fid.seek(0 + offset, SEEK_SET)
        val = unpack(fmt, fid.read(calcsize(fmt)))
        if len(val) == 1:
            header[key] = val[0]
        else:
            header[key] = val

    # sections
    sections = {}
    for s, sectionName in enumerate(sectionNames):
        fid.seek(76 + s * 16)
        uBlockIndex, uBytes, llNumEntries = unpack('IIl', fid.read(calcsize('IIl')))
        sections[sectionName] = {}
        sections[sectionName]['uBlockIndex'] = uBlockIndex
        sections[sectionName]['uBytes'] = uBytes
        sections[sectionName]['llNumEntries'] = llNumEntries
    header['sections'] = sections

    # strings sections
    # hack for reading channels names and units
    fid.seek(sections['StringsSection']['uBlockIndex'] * BLOCKSIZE)
    big_string = fid.read(sections['StringsSection']['uBytes'])
    goodstart = -1
    for key in [b'AXENGN', b'clampex', b'Clampex', b'CLAMPEX', b'axoscope', b'Clampfit']:
        goodstart = big_string.find(key)
        if goodstart != -1:
            break
    assert goodstart != -1, 'This file does not contain clampex, axoscope or clampfit in the header'
    big_string = big_string[goodstart:]
    strings = big_string.split(b'\x00')

    # ADC sections
    header['listADCInfo'] = []
    for i in range(sections['ADCSection']['llNumEntries']):
        # read ADCInfo
        fid.seek(sections['ADCSection']['uBlockIndex'] *
                 BLOCKSIZE + sections['ADCSection']['uBytes'] * i)
        ADCInfo = _read_info_as_dict(fid, ADCInfoDescription)
        ADCInfo['ADCChNames'] = strings[ADCInfo['lADCChannelNameIndex'] - 1]
        ADCInfo['ADCChUnits'] = strings[ADCInfo['lADCUnitsIndex'] - 1]
        header['listADCInfo'].append(ADCInfo)

    # protocol sections
    fid.seek(sections['ProtocolSection']['uBlockIndex'] * BLOCKSIZE)
    header['protocol'] = _read_info_as_dict(fid, protocolInfoDescription)
    header['sProtocolPath'] = strings[header['uProtocolPathIndex'] - 1]

    # DAC sections
    header['listDACInfo'] = []
    for i in range(sections['DACSection']['llNumEntries']):
        # read DACInfo
        fid.seek(sections['DACSection']['uBlockIndex'] *
                 BLOCKSIZE + sections['DACSection']['uBytes'] * i)
        DACInfo = _read_info_as_dict(fid, DACInfoDescription)
        DACInfo['DACChNames'] = strings[DACInfo['lDACChannelNameIndex'] - 1]
        DACInfo['DACChUnits'] = strings[
            DACInfo['lDACChannelUnitsIndex'] - 1]

        header['listDACInfo'].append(DACInfo)

    """ Not present in test file. No tests, no code.
    # tags
    listTag = []
    for i in range(sections['TagSection']['llNumEntries']):
        fid.seek(sections['TagSection']['uBlockIndex'] *
                 BLOCKSIZE + sections['TagSection']['uBytes'] * i)
        tag = _read_info_as_dict(fid, TagInfoDescription)
        listTag.append(tag)

    header['listTag'] = listTag

    # EpochPerDAC  sections
    # header['dictEpochInfoPerDAC'] is dict of dicts:
    #  - the first index is the DAC number
    #  - the second index is the epoch number
    # It has to be done like that because data may not exist
    # and may not be in sorted order
    header['dictEpochInfoPerDAC'] = {}
    for i in range(sections['EpochPerDACSection']['llNumEntries']):
        #  read DACInfo
        fid.seek(sections['EpochPerDACSection']['uBlockIndex'] *
                 BLOCKSIZE +
                 sections['EpochPerDACSection']['uBytes'] * i)
        EpochInfoPerDAC = _read_info_as_dict(fid, EpochInfoPerDACDescription)
        DACNum = EpochInfoPerDAC['nDACNum']
        EpochNum = EpochInfoPerDAC['nEpochNum']
        # Checking if the key exists, if not, the value is empty
        # so we have to create empty dict to populate
        if DACNum not in header['dictEpochInfoPerDAC']:
            header['dictEpochInfoPerDAC'][DACNum] = {}

        header['dictEpochInfoPerDAC'][DACNum][EpochNum] =\
            EpochInfoPerDAC
    """

    return header
1249725
function readDirectory (srcDir) {
  let srcTree = {}
  const entries = fs.readdirSync(srcDir)

  entries.forEach(function (entry) {
    const filePath = path.join(srcDir, entry)
    if (fs.lstatSync(filePath).isDirectory()) {
      srcTree[entry] = readDirectory(filePath)
    } else {
      srcTree[entry] = fs.readFileSync(filePath, {
        encoding: 'utf8'
      })
    }
  })

  return srcTree
}
720076
def read_fastas(input_files):
    
    tumor_file = [y for x, y in input_files.items() if x.startswith('T')][0]
    normal_file = [y for x, y in input_files.items() if x.startswith('N')][0]
    output_files = defaultdict(list)
    output_files = _read_fasta(tumor_file, output_files)
    num_entries = len(output_files)
    output_files = _read_fasta(normal_file, output_files)
    assert len(output_files) == num_entries
    return output_files
545742
def read_header(filename):
    '''  '''
    header = {}
    in_header = False
    data = nl.universal_read(filename)
    lines = [x.strip() for x in data.split('\n')]
    for line in lines:
        if line=="*** Header Start ***":
            in_header=True
            continue
        if line=="*** Header End ***":
            return header
        fields = line.split(": ")
        if len(fields)==2:
            header[fields[0]] = fields[1]
25000
def _read_fasta(fasta_file, output_dict):
    
    read_name = None
    with open(fasta_file, 'r') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            if line.startswith('>'):
                read_name = line.lstrip('>')
            else:
                assert read_name is not None, line
                output_dict[read_name].append(line.strip())
    return output_dict
757158
def find_all(self, header, list_type=None):
        """"""
        found = []
        for chunk in self:
            if chunk.header == header and (not list_type or (header in
                list_headers and chunk.type == list_type)):
                found.append(chunk)
        return found
452628
def read_header(cls, headers_path, block_height, allow_none=False):
        
        if os.path.exists(headers_path):
    
            header_parser = BlockHeaderSerializer()
            sb = os.stat( headers_path )
            if sb.st_size < BLOCK_HEADER_SIZE * block_height:
                # beyond EOF 
                if allow_none:
                    return None 
                else:
                    raise Exception('EOF on block headers')

            with open( headers_path, "rb" ) as f:
                f.seek( block_height * BLOCK_HEADER_SIZE, os.SEEK_SET )
                hdr = SPVClient.read_header_at( f )

            return hdr
        else:
            if allow_none:
                return None
            else:
                raise Exception('No such file or directory: {}'.format(headers_path))
463453
def read_fastq(filename):
    
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3:
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh)
841739
def readall(self):
        # type: () -> bytes
        '''
        
        '''
        if not self._open:
            raise pycdlibexception.PyCdlibInvalidInput('I/O operation on closed file.')

        readsize = self._length - self._offset
        if readsize > 0:
            data = self._fp.read(readsize)
            self._offset += readsize
        else:
            data = b''

        return data
296685
def fastp_filtered_reads_chart(self):
        
        # Specify the order of the different possible categories
        keys = OrderedDict()
        keys['filtering_result_passed_filter_reads'] =  { 'name': 'Passed Filter' }
        keys['filtering_result_low_quality_reads'] =    { 'name': 'Low Quality' }
        keys['filtering_result_too_many_N_reads'] =     { 'name': 'Too Many N' }
        keys['filtering_result_too_short_reads'] =      { 'name': 'Too short' }

        # Config for the plot
        pconfig = {
            'id': 'fastp_filtered_reads_plot',
            'title': 'Fastp: Filtered Reads',
            'ylab': '# Reads',
            'cpswitch_counts_label': 'Number of Reads',
            'hide_zero_cats': False,
        }
        return bargraph.plot(self.fastp_data, keys, pconfig)
657264
def are_imaging_dicoms(dicom_input):
    

    # if it is philips and multiframe dicom then we assume it is ok
    if common.is_philips(dicom_input):
        if common.is_multiframe_dicom(dicom_input):
            return True

    # for all others if there is image position patient we assume it is ok
    header = dicom_input[0]
    return Tag(0x0020, 0x0037) in header
1204712
def tryload_cache_list(dpath, fname, cfgstr_list, verbose=False):
    
    data_list = [tryload_cache(dpath, fname, cfgstr, verbose) for cfgstr in cfgstr_list]
    ismiss_list = [data is None for data in data_list]
    return data_list, ismiss_list
1126291
def _remove_localizers_by_imagetype(dicoms):
    
    # Loop overall files and build dict
    filtered_dicoms = []
    for dicom_ in dicoms:
        if 'ImageType' in dicom_ and 'LOCALIZER' in dicom_.ImageType:
            continue
        # 'Projection Image' are Localizers for CT only see MSMET-234
        if 'CT' in dicom_.Modality and 'ImageType' in dicom_ and 'PROJECTION IMAGE' in dicom_.ImageType:
            continue
        filtered_dicoms.append(dicom_)
    return filtered_dicoms