Extracts tuples representing comments and localization entries from strings file.

    Args:
        file_descriptor (file): The file to read the tuples from

    Returns:
        list : List of tuples representing the headers and localization entries.
350581
def extract_header_comment_key_value_tuples_from_file(file_descriptor):
    
    file_data = file_descriptor.read()
    findall_result = re.findall(HEADER_COMMENT_KEY_VALUE_TUPLES_REGEX, file_data, re.MULTILINE | re.DOTALL)

    returned_list = []
    for header_comment, _ignored, raw_comments, key, value in findall_result:
        comments = re.findall("/\* (.*?) \*/", raw_comments)
        if len(comments) == 0:
            comments = [u""]
        returned_list.append((header_comment, comments, key, value))

    return returned_list
799862
def from_file(cls, f):
        
        word2index = {}
        counts = Counter()
        for i, line in enumerate(f):
            word, count_str = line.split('\t')
            word = word.decode('utf-8')
            word2index[word] = i
            counts[word] = float(count_str)
            if i == 0:
                unk = word
        return cls.from_dict(word2index, unk, counts)
155437
def method_descriptor(descriptor: str) -> MethodDescriptor:
    
    end_para = descriptor.find(')')
    returns = descriptor[end_para + 1:]
    args = descriptor[1:end_para]

    return MethodDescriptor(
        parse_descriptor(returns)[0],
        parse_descriptor(args),
        returns,
        args,
        descriptor
    )
747086
def _list_getter(self, uri, headers, last_obj=None, spr=False):
        

        # Quote the file path.
        base_path = marked_path = ('%s?limit=10000&format=json' % uri.path)

        if last_obj:
            marked_path = self._last_marker(
                base_path=base_path,
                last_object=cloud_utils.quoter(last_obj)
            )

        file_list = self._obj_index(
            uri=uri,
            base_path=base_path,
            marked_path=marked_path,
            headers=headers,
            spr=spr
        )

        LOG.debug(
            'Found [ %d ] entries(s) at [ %s ]',
            len(file_list),
            uri.geturl()
        )

        if spr:
            return file_list
        else:
            return cloud_utils.unique_list_dicts(
                dlist=file_list, key='name'
            )
319712
def read_dict(file_name, clear_none=False, encoding='utf-8'):
    
    with open(file_name, 'rb') as f:
        data = f.read()

    if encoding is not None:
        data = data.decode(encoding)

    line_list = data.splitlines()
    data = []
    i = 0
    for line in line_list:
        i += 1
        try:
            line = force_text(line).strip()
            data.append(line)
        except:
            print('read error line %s' % i)
    if clear_none:
        data = [t for t in data if t != '']
    data = deque(data)
    return data
1262579
def headers_to_str_headers(headers):
    '''
    
    '''
    ret = []

    if isinstance(headers, collections_abc.Mapping):
        h = headers.items()
    else:
        h = headers

    if six.PY2:  #pragma: no cover
        return h

    for tup in h:
        k, v = tup
        if isinstance(k, six.binary_type):
            k = k.decode('iso-8859-1')
        if isinstance(v, six.binary_type):
            v = v.decode('iso-8859-1')
        ret.append((k, v))
    return ret
1315798
def comments_of_file(file)
      File.readlines(file).map { |s| s[2..-1].rstrip if s.start_with?('# ') }.compact
    end
913231
def describe_file(module):
    
    descriptor = FileDescriptor()
    descriptor.package = util.get_package_for_module(module)

    if not descriptor.package:
        descriptor.package = None

    message_descriptors = []
    enum_descriptors = []

    # Need to iterate over all top level attributes of the module looking for
    # message and enum definitions.  Each definition must be itself described.
    for name in sorted(dir(module)):
        value = getattr(module, name)

        if isinstance(value, type):
            if issubclass(value, messages.Message):
                message_descriptors.append(describe_message(value))

            elif issubclass(value, messages.Enum):
                enum_descriptors.append(describe_enum(value))

    if message_descriptors:
        descriptor.message_types = message_descriptors

    if enum_descriptors:
        descriptor.enum_types = enum_descriptors

    return descriptor
1364172
def _read_fd(file_descr):
        
        try:
            line = os.read(file_descr, 1024 * 1024)
        except OSError:
            stream_desc = NonBlockingStreamReader._get_sd(file_descr)
            if stream_desc is not None:
                stream_desc.has_error = True
                if stream_desc.callback is not None:
                    stream_desc.callback()
            return 0

        if line:
            stream_desc = NonBlockingStreamReader._get_sd(file_descr)
            if stream_desc is None:
                return 0 # Process closing

            if IS_PYTHON3:
                try:
                    # @TODO: further develop for not ascii/unicode binary content
                    line = line.decode("ascii")
                except UnicodeDecodeError:
                    line = repr(line)
            stream_desc.buf += line
            # Break lines
            split = stream_desc.buf.split(os.linesep)
            for line in split[:-1]:
                stream_desc.read_queue.appendleft(strip_escape(line.strip()))
                if stream_desc.callback is not None:
                    stream_desc.callback()
            # Store the remainded, its either '' if last char was '\n'
            # or remaining buffer before line end
            stream_desc.buf = split[-1]
            return len(line)
        return 0
936878
def _load_unicode_block_info(self):
        
        filename = "Blocks.txt"
        current_dir = os.path.abspath(os.path.dirname(__file__))
        with codecs.open(os.path.join(current_dir, filename), mode="r", encoding="utf-8") as fp:
            for line in fp:
                if not line.strip() or line.startswith("#"):
                    continue  # Skip empty lines or lines that are comments (comments start with '#')
                # Format: Start Code..End Code; Block Name
                block_range, block_name = line.strip().split(";")
                start_range, end_range = block_range.strip().split("..")
                self._unicode_blocks[six.moves.range(int(start_range, 16), int(end_range, 16) + 1)] = block_name.strip()
1092198
def getASCIIStringDescriptor(self, descriptor, errors='strict'):
        
        if descriptor == 0:
            return None
        descriptor_string = bytearray(STRING_LENGTH)
        try:
            received = mayRaiseUSBError(libusb1.libusb_get_string_descriptor_ascii(
                self.__handle, descriptor,
                create_binary_buffer(descriptor_string)[0],
                STRING_LENGTH,
            ))
        # pylint: disable=undefined-variable
        except USBErrorNotFound:
            # pylint: enable=undefined-variable
            return None
        return descriptor_string[:received].decode('ASCII', errors=errors)
256805
def DoxyfileParse(file_contents):
   
   data = {}

   import shlex
   lex = shlex.shlex(instream = file_contents, posix = True)
   lex.wordchars += "*+./-:"
   lex.whitespace = lex.whitespace.replace("\n", "")
   lex.escape = ""

   lineno = lex.lineno
   token = lex.get_token()
   key = token   # the first token should be a key
   last_token = ""
   key_token = False
   next_key = False
   new_data = True

   def append_data(data, key, new_data, token):
      if new_data or len(data[key]) == 0:
         data[key].append(token)
      else:
         data[key][-1] += token

   while token:
      if token in ['\n']:
         if last_token not in ['\\']:
            key_token = True
      elif token in ['\\']:
         pass
      elif key_token:
         key = token
         key_token = False
      else:
         if token == "+=":
            if not data.has_key(key):
               data[key] = list()
         elif token == "=":
            if key == "TAGFILES" and data.has_key(key):
               append_data( data, key, False, "=" )
               new_data=False
            else:
               data[key] = list()
         else:
            append_data( data, key, new_data, token )
            new_data = True

      last_token = token
      token = lex.get_token()

      if last_token == '\\' and token != '\n':
         new_data = False
         append_data( data, key, new_data, '\\' )

   # compress lists of len 1 into single strings
   for (k, v) in data.items():
      if len(v) == 0:
         data.pop(k)

      # items in the following list will be kept as lists and not converted to strings
      if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS", "TAGFILES"]:
         continue

      if len(v) == 1:
         data[k] = v[0]

   return data
61687
def extract_headers(headers):
    """"""
    sorted_headers = {}
    matches = re.findall(r'(.*):\s(.*)', headers)
    for match in matches:
        header = match[0]
        value = match[1]
        try:
            if value[-1] == ',':
                value = value[:-1]
            sorted_headers[header] = value
        except IndexError:
            pass
    return sorted_headers
621026
def __parse(self, lines):
        

        # Every line in the file must consist of either a comment
        # or a key-value pair. A key-value pair is a line consisting
        # of a key which is a combination of non-white space characters
        # The separator character between key-value pairs is a '=',
        # ':' or a whitespace character not including the newline.
        # If the '=' or ':' characters are found, in the line, even
        # keys containing whitespace chars are allowed.

        # A line with only a key according to the rules above is also
        # fine. In such case, the value is considered as the empty string.
        # In order to include characters '=' or ':' in a key or value,
        # they have to be properly escaped using the backslash character.

        # Some examples of valid key-value pairs:
        #
        # key     value
        # key=value
        # key:value
        # key     value1,value2,value3
        # key     value1,value2,value3 \
        #         value4, value5
        # key
        # This key= this value
        # key = value1 value2 value3

        # Any line that starts with a '#' or '!' is considerered a comment
        # and skipped. Also any trailing or preceding whitespaces
        # are removed from the key/value.

        # This is a line parser. It parses the
        # contents like by line.

        lineno=0
        i = iter(lines)

        for line in i:
            lineno += 1
            line = line.strip()
            # Skip null lines
            if not line: continue
            # Skip lines which are comments
            if line[0] in ('#','!'): continue
            # Some flags
            escaped=False
            # Position of first separation char
            sepidx = -1
            # A flag for performing wspace re check
            flag = 0
            # Check for valid space separation
            # First obtain the max index to which we
            # can search.
            m = self.othercharre.search(line)
            if m:
                first, last = m.span()
                start, end = 0, first
                flag = 1
                wspacere = re.compile(r'(?<![\\\=\:])(\s)')
            else:
                if self.othercharre2.search(line):
                    # Check if either '=' or ':' is present
                    # in the line. If they are then it means
                    # they are preceded by a backslash.

                    # This means, we need to modify the
                    # wspacere a bit, not to look for
                    # : or = characters.
                    wspacere = re.compile(r'(?<![\\])(\s)')
                start, end = 0, len(line)

            m2 = wspacere.search(line, start, end)
            if m2:
                # print 'Space match=>',line
                # Means we need to split by space.
                first, last = m2.span()
                sepidx = first
            elif m:
                # print 'Other match=>',line
                # No matching wspace char found, need
                # to split by either '=' or ':'
                first, last = m.span()
                sepidx = last - 1
                # print line[sepidx]


            # If the last character is a backslash
            # it has to be preceded by a space in which
            # case the next line is read as part of the
            # same property
            while line[-1] == '\\':
                # Read next line
                nextline = next(i)
                nextline = nextline.strip()
                lineno += 1
                # This line will become part of the value
                line = line[:-1] + nextline

            # Now split to key,value according to separation char
            if sepidx != -1:
                key, value = line[:sepidx], line[sepidx+1:]
            else:
                key,value = line,''
            self._keyorder.append(key)
            self.process_pair(key, value)
315010
def from_files(cls, directory: str) -> 'Vocabulary':
        
        logger.info("Loading token dictionary from %s.", directory)
        with codecs.open(os.path.join(directory, NAMESPACE_PADDING_FILE), 'r', 'utf-8') as namespace_file:
            non_padded_namespaces = [namespace_str.strip() for namespace_str in namespace_file]

        vocab = cls(non_padded_namespaces=non_padded_namespaces)

        # Check every file in the directory.
        for namespace_filename in os.listdir(directory):
            if namespace_filename == NAMESPACE_PADDING_FILE:
                continue
            if namespace_filename.startswith("."):
                continue
            namespace = namespace_filename.replace('.txt', '')
            if any(namespace_match(pattern, namespace) for pattern in non_padded_namespaces):
                is_padded = False
            else:
                is_padded = True
            filename = os.path.join(directory, namespace_filename)
            vocab.set_from_file(filename, is_padded, namespace=namespace)

        return vocab
407871
def extract_descriptor(self, obj):
        """"""
        descriptor = []

        def flatten(current):
            """Flatten descriptor."""
            if isinstance(current, dict):
                for key in current:
                    flatten(current[key])
            elif isinstance(current, list):
                for val in current:
                    flatten(val)
            elif isinstance(current, (int, bool, float, str)):
                descriptor.append(str(current))

        flatten(obj.descriptor)

        return descriptor
1269173
def extract_headers(lines, max_wrap_lines):
    
    hdrs = {}
    header_name = None

    # Track overlong headers that extend over multiple lines
    extend_lines = 0

    lines_processed = 0

    for n, line in enumerate(lines):
        if not line.strip():
            header_name = None
            continue

        match = HEADER_RE.match(line)
        if match:
            header_name, header_value = match.groups()
            header_name = header_name.strip().lower()
            extend_lines = 0

            if header_name in HEADER_MAP:
                hdrs[HEADER_MAP[header_name]] = header_value.strip()
            lines_processed = n+1
        else:
            extend_lines += 1
            if extend_lines < max_wrap_lines and header_name in HEADER_MAP:
                hdrs[HEADER_MAP[header_name]] = join_wrapped_lines(
                        [hdrs[HEADER_MAP[header_name]], line.strip()])
                lines_processed = n+1
            else:
                # no more headers found
                break

    return hdrs, lines_processed