Parse an input stream containing a Fortran namelist.
748
def _readstream(self, nml_file, nml_patch_in=None):
        """"""
        nml_patch = nml_patch_in if nml_patch_in is not None else Namelist()

        tokenizer = Tokenizer()
        f90lex = []
        for line in nml_file:
            toks = tokenizer.parse(line)
            while tokenizer.prior_delim:
                new_toks = tokenizer.parse(next(nml_file))

                # Skip empty lines
                if not new_toks:
                    continue

                # The tokenizer always pre-tokenizes the whitespace (leftover
                # behaviour from Fortran source parsing) so this must be added
                # manually.
                if new_toks[0].isspace():
                    toks[-1] += new_toks.pop(0)

                # Append the rest of the string (if present)
                if new_toks:
                    toks[-1] += new_toks[0]

                    # Attach the rest of the tokens
                    toks.extend(new_toks[1:])

            toks.append('\n')
            f90lex.extend(toks)

        self.tokens = iter(f90lex)

        nmls = Namelist()

        # Attempt to get first token; abort on empty file
        try:
            self._update_tokens(write_token=False)
        except StopIteration:
            return nmls

        # TODO: Replace "while True" with an update_token() iterator
        while True:
            try:
                # Check for classic group terminator
                if self.token == 'end':
                    self._update_tokens()

                # Ignore tokens outside of namelist groups
                while self.token not in ('&', '$'):
                    self._update_tokens()

            except StopIteration:
                break

            # Create the next namelist
            self._update_tokens()
            g_name = self.token

            g_vars = Namelist()
            v_name = None

            # TODO: Edit `Namelist` to support case-insensitive `get` calls
            grp_patch = nml_patch.get(g_name.lower(), Namelist())

            # Populate the namelist group
            while g_name:

                if self.token not in ('=', '%', '('):
                    self._update_tokens()

                # Set the next active variable
                if self.token in ('=', '(', '%'):

                    v_name, v_values = self._parse_variable(
                        g_vars,
                        patch_nml=grp_patch
                    )

                    if v_name in g_vars:
                        v_prior_values = g_vars[v_name]
                        v_values = merge_values(v_prior_values, v_values)

                    g_vars[v_name] = v_values

                    # Deselect variable
                    v_name = None
                    v_values = []

                # Finalise namelist group
                if self.token in ('/', '&', '$'):

                    # Append any remaining patched variables
                    for v_name, v_val in grp_patch.items():
                        g_vars[v_name] = v_val
                        v_strs = nmls._var_strings(v_name, v_val)
                        for v_str in v_strs:
                            self.pfile.write(
                                '{0}{1}\n'.format(nml_patch.indent, v_str)
                            )

                    # Append the grouplist to the namelist
                    if g_name in nmls:
                        g_update = nmls[g_name]

                        # Update to list of groups
                        if not isinstance(g_update, list):
                            g_update = [g_update]

                        g_update.append(g_vars)

                    else:
                        g_update = g_vars

                    nmls[g_name] = g_update

                    # Reset state
                    g_name, g_vars = None, None

            try:
                self._update_tokens()
            except StopIteration:
                break

        return nmls
943152
def parse(self, stream):
        """"""
        yaml = self._load_yaml()

        try:
            parsed_obj = yaml.safe_load(stream)
        except Exception as e:
            raise ConfigFileParserException("Couldn't parse config file: %s" % e)

        if not isinstance(parsed_obj, dict):
            raise ConfigFileParserException("The config file doesn't appear to "
                "contain 'key: value' pairs (aka. a YAML mapping). "
                "yaml.load('%s') returned type '%s' instead of 'dict'." % (
                getattr(stream, 'name', 'stream'),  type(parsed_obj).__name__))

        result = OrderedDict()
        for key, value in parsed_obj.items():
            if isinstance(value, list):
                result[key] = value
            else:
                result[key] = str(value)

        return result
724203
def p_namelist(self,t):
    ""
    if len(t)==2: t[0] = [t[1]]
    elif len(t)==4: t[0] = t[1] + [t[3]]
    else: raise NotImplementedError('unk_len',len(t)) # pragma: no cover
925391
def __parse(self) -> object:
        """"""
        char = self.data[self.idx: self.idx + 1]
        if char in [b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0']:
            str_len = int(self.__read_to(b':'))
            return self.__read(str_len)
        elif char == b'i':
            self.idx += 1
            return int(self.__read_to(b'e'))
        elif char == b'd':
            return self.__parse_dict()
        elif char == b'l':
            return self.__parse_list()
        elif char == b'':
            raise bencodepy.DecodingError('Unexpected End of File at index position of {0}.'.format(str(self.idx)))
        else:
            raise bencodepy.DecodingError(
                'Invalid token character ({0}) at position {1}.'.format(str(char), str(self.idx)))
62277
def parse_names(cls, names: List[str]) -> T:
        """"""
        value = 0
        iterable = cls  # type: Iterable
        for name in names:
            name = name.lower()
            flag = next((item for item in iterable if name == item.name.lower()), None)
            if not flag:
                raise ValueError("{} is not a member of {}".format(
                    name, cls.__name__))
            value = value | int(flag)
        return cls(value)
299952
def from_stream(cls, stream, offset):
        
        ifd_parser = _IfdParser(stream, offset)
        entries = dict((e.tag, e.value) for e in ifd_parser.iter_entries())
        return cls(entries)
815969
def parse(fin)
      # Headline
      names = fin.gets.chomp.split("\t")
      index2identifier = {}
      index_hgncid = nil
      BioTCM::Databases::HGNC::IDENTIFIERS.each do |identifer, name|
        if identifer == :hgncid
          index_hgncid = names.index(name)
        elsif name.is_a?(String)
          index2identifier[names.index(name)] = identifer if names.index(name)
        else
          name.each_with_index do |n, i|
            next unless names.index(n)
            index2identifier[names.index(n)] = (i == 0 ? identifer : identifer.to_s)
          end
        end
      end

      # Dynamically bulid a line processor
      process_one_line = index2identifier.collect do |index, identifer|
        # Symbol will be mapped to single item
        if identifer.is_a?(Symbol)
          %(
            unless column[#{index}] == nil || column[#{index}] == "" || column[#{index}] == "-"
              @#{identifer}2hgncid[column[#{index}]] = column[#{index_hgncid}]
              @hgncid2#{identifer}[column[#{index_hgncid}]] = column[#{index}]
            end
          )
        # Others will be mapped to list item
        else
          %{
            unless column[#{index}] == nil
              column[#{index}].split(", ").each do |id|
          } +
            if identifer == 'symbol'
              %(
                if @ambiguous_symbol[id]
                  @ambiguous_symbol[id] << @hgncid2symbol[column[#{index_hgncid}]]
                elsif @symbol2hgncid[id].nil?
                  @symbol2hgncid[id] = column[#{index_hgncid}]
                else
                  @ambiguous_symbol[id] = [@hgncid2symbol[column[#{index_hgncid}]]]
                  unless @hgncid2symbol[@symbol2hgncid[id]] == id
                    @ambiguous_symbol[id] << @hgncid2symbol[@symbol2hgncid[id]]
                    @symbol2hgncid.delete(id)
                  end
                end
              )
            else
              %(
                @#{identifer}2hgncid[id] = column[#{index_hgncid}] if @#{identifer}2hgncid[id].nil?
              )
            end +
            %(
              end
            end
            )
        end
      end.join

      # Process the content
      eval %{fin.each do |line|\n column = line.chomp.split("\\t", -1)} + process_one_line + 'end' # rubocop:disable Lint/Eval
    end
204074
def from_stream(cls, stream):
        
        chunk_parser = _ChunkParser.from_stream(stream)
        chunks = [chunk for chunk in chunk_parser.iter_chunks()]
        return cls(chunks)
48568
def _lex_fortran(self, match, ctx=None):
        """"""
        lexer = FortranLexer()
        text = match.group(0) + "\n"
        for index, token, value in lexer.get_tokens_unprocessed(text):
            value = value.replace('\n', '')
            if value != '':
                yield index, token, value
141642
private InputStream parseEntity(DataSource ds) {
        try {
            return ds.getInputStream();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
335457
def parse_streams(self):
        
        logger.debug("Parsing streams of {}".format(self.input_file))

        cmd = [
            self.ffmpeg_normalize.ffmpeg_exe, '-i', self.input_file,
            '-c', 'copy', '-t', '0', '-map', '0',
            '-f', 'null', NUL
        ]

        cmd_runner = CommandRunner(cmd)
        cmd_runner.run_command()
        output = cmd_runner.get_output()

        logger.debug("Stream parsing command output:")
        logger.debug(output)

        output_lines = [line.strip() for line in output.split('\n')]

        for line in output_lines:

            if not line.startswith('Stream'):
                continue

            stream_id_match = re.search(r'#0:([\d]+)', line)
            if stream_id_match:
                stream_id = int(stream_id_match.group(1))
                if stream_id in self._stream_ids():
                    continue
            else:
                continue

            if 'Audio' in line:
                logger.debug("Found audio stream at index {}".format(stream_id))
                sample_rate_match = re.search(r'(\d+) Hz', line)
                sample_rate = int(sample_rate_match.group(1)) if sample_rate_match else None
                bit_depth_match = re.search(r's(\d+)p?,', line)
                bit_depth = int(bit_depth_match.group(1)) if bit_depth_match else None
                self.streams['audio'][stream_id] = AudioStream(self, stream_id, sample_rate, bit_depth)

            elif 'Video' in line:
                logger.debug("Found video stream at index {}".format(stream_id))
                self.streams['video'][stream_id] = VideoStream(self, stream_id)

            elif 'Subtitle' in line:
                logger.debug("Found subtitle stream at index {}".format(stream_id))
                self.streams['subtitle'][stream_id] = SubtitleStream(self, stream_id)

        if not self.streams['audio']:
            raise FFmpegNormalizeError(
                "Input file {} does not contain any audio streams"
                .format(self.input_file))

        if os.path.splitext(self.output_file)[1].lower() in ['.wav', '.mp3', '.aac']:
            logger.warning(
                "Output file only supports one stream. "
                "Keeping only first audio stream."
            )
            first_stream = list(self.streams['audio'].values())[0]
            self.streams['audio'] = {first_stream.stream_id: first_stream}
            self.streams['video'] = {}
            self.streams['subtitle'] = {}
321770
def _parse_literal_list(self, indent):
        """"""
        if self._cur_token['type'] not in self._literals:
            raise Exception(
                "Parser failed, _parse_literal_list was called on non-literal"
                " {} on line {}.".format(
                    repr(self._cur_token['value']), self._cur_token['line']
                )
            )

        # find next token after whitespace without incrementing
        temp_position = self._cur_position
        while (
            temp_position < self.num_tokens-1 and (
                self.tokens[temp_position]['type'] is TT.ws or
                self.tokens[temp_position]['type'] in self._literals
            )
        ):
            temp_position += 1
        next_token = self.tokens[temp_position]

        # end of stream
        if next_token['type'] is TT.ws:
            return self._cur_token['value']
        elif next_token['type'] is TT.comma:
            return self._parse_comma_list()
        elif next_token['type'] is TT.lbreak:
            while (
                temp_position < self.num_tokens-1 and
                self.tokens[temp_position]['type'] in (TT.lbreak, TT.ws)
            ):
                temp_position += 1
            if self.tokens[temp_position]['type'] in self._literals:
                return self._parse_newline_list(indent)
            else:
                rval = self._cur_token['value']
                self._increment()
                return rval
        else:
            rval = self._cur_token['value']
            self._increment()
            return rval
456908
def __parse_stream(self, stream):
        """"""

        nline = 0
        lines = stream.split('\n')

        for line in lines:
            nline += 1

            # Ignore blank lines and comments
            m = re.match(self.LINES_TO_IGNORE_REGEX, line, re.UNICODE)
            if m:
                continue

            line = line.strip('\n').strip(' ')
            parts = line.split('>')

            if len(parts) == 0:
                cause = "line %s: invalid format" % str(nline)
                raise InvalidFormatError(cause=cause)

            aliases = []

            for part in parts:
                part = part.replace(',', ' ')
                part = part.strip('\n').strip(' ')

                if len(part) == 0:
                    continue

                if part.find('<') < 0:
                    cause = "line %s: invalid format" % str(nline)
                    raise InvalidFormatError(cause=cause)

                alias = email.utils.parseaddr(part + '>')
                aliases.append(alias)

            yield aliases
581197
public List<T> parseList(InputStream is) throws IOException {
        JsonParser jsonParser = LoganSquare.JSON_FACTORY.createParser(is);
        jsonParser.nextToken();
        return parseList(jsonParser);
    }
1232843
def parse():
    """"""
    parser = argparse.ArgumentParser()

    parser.add_argument('--version', action='version',
                        version='f90nml {0}'.format(f90nml.__version__))

    parser.add_argument('--group', '-g', action='store',
                        help="specify namelist group to modify. "
                        "When absent, the first group is used")
    parser.add_argument('--variable', '-v', action='append',
                        help="specify the namelist variable to add or modify, "
                        "followed by the new value. Expressions are of the "
                        "form `VARIABLE=VALUE`")
    parser.add_argument('--patch', '-p', action='store_true',
                        help="modify the existing namelist as a patch")
    parser.add_argument('--format', '-f', action='store',
                        help="specify the output format (json, yaml, or nml)")
    parser.add_argument('--output', '-o', action='store',
                        help="specify namelist group to modify. "
                        "When absent, the first group is used")

    parser.add_argument('input', nargs='?')
    parser.add_argument('output', nargs='?')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit()

    args = parser.parse_args()

    input_fname = args.input
    output_fname = args.output

    # Get input format
    # TODO: Combine with output format
    if input_fname:
        _, input_ext = os.path.splitext(input_fname)
        if input_ext == '.json':
            input_fmt = 'json'
        elif input_ext == '.yaml':
            input_fmt = 'yaml'
        else:
            input_fmt = 'nml'
    else:
        input_fmt = 'nml'

    # Output format flag validation
    valid_formats = ('json', 'yaml', 'nml')
    if args.format and args.format not in valid_formats:
        print('f90nml: error: format must be one of the following: {0}'
              ''.format(valid_formats), file=sys.stderr)
        sys.exit(-1)

    # Get output format
    # TODO: Combine with input format
    if not args.format:
        if output_fname:
            _, output_ext = os.path.splitext(output_fname)
            if output_ext == '.json':
                output_fmt = 'json'
            elif output_ext in ('.yaml', '.yml'):
                output_fmt = 'yaml'
            else:
                output_fmt = 'nml'
        else:
            output_fmt = 'nml'
    else:
        output_fmt = args.format

    # Confirm that YAML module is available
    if (input_fmt == 'yaml' or output_fmt == 'yaml') and not has_yaml:
        print('f90nml: error: YAML module could not be found.',
              file=sys.stderr)
        sys.exit(-1)

    # Do not patch non-namelist output
    if any(fmt != 'nml' for fmt in (input_fmt, output_fmt)) and args.patch:
        print('f90nml: error: Only namelist files can be patched.',
              file=sys.stderr)
        sys.exit(-1)

    # Read the input file
    if input_fname:
        if input_fmt in ('json', 'yaml'):
            if input_fmt == 'json':
                with open(input_fname) as input_file:
                    input_data = json.load(input_file)
            elif input_ext == '.yaml':
                with open(input_fname) as input_file:
                    input_data = yaml.safe_load(input_file)
        else:
            input_data = f90nml.read(input_fname)
    else:
        input_data = {}

    input_data = f90nml.Namelist(input_data)

    # Construct the update namelist
    update_nml = {}
    if args.variable:
        if not args.group:
            # Use the first available group
            grp = list(input_data.keys())[0]
            warnings.warn('f90nml: warning: Assuming variables are in group \'{0}\'.'.format(grp))
        else:
            grp = args.group

        update_nml_str = '&{0} {1} /\n'.format(grp, ', '.join(args.variable))
        update_io = StringIO(update_nml_str)
        update_nml = f90nml.read(update_io)
        update_io.close()

    # Target output
    output_file = open(output_fname, 'w') if output_fname else sys.stdout

    if args.patch:
        # We have to read the file twice for a patch.  The main reason is
        # to identify the default group, in case this is not provided.
        # It could be avoided if a group is provided, but logically that could
        # a mess that I do not want to sort out right now.
        f90nml.patch(input_fname, update_nml, output_file)

    else:
        # Update the input namelist directly
        if update_nml:
            try:
                input_data[grp].update(update_nml[grp])
            except KeyError:
                input_data[grp] = update_nml[grp]

        # Write to output
        if not args.patch:
            if output_fmt in ('json', 'yaml'):
                if output_fmt == 'json':
                    input_data = input_data.todict(complex_tuple=True)
                    json.dump(input_data, output_file,
                              indent=4, separators=(',', ': '))
                    output_file.write('\n')

                elif output_fmt == 'yaml':
                    input_data = input_data.todict(complex_tuple=True)
                    yaml.dump(input_data, output_file,
                              default_flow_style=False)
            else:
                # Default to namelist output
                f90nml.write(input_data, output_file)

    # Cleanup
    if output_file != sys.stdout:
        output_file.close()
934844
def parse_if(self):
        """"""
        node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
        while 1:
            node.test = self.parse_tuple(with_condexpr=False)
            node.body = self.parse_statements(('name:elif', 'name:else',
                                               'name:endif'))
            node.elif_ = []
            node.else_ = []
            token = next(self.stream)
            if token.test('name:elif'):
                node = nodes.If(lineno=self.stream.current.lineno)
                result.elif_.append(node)
                continue
            elif token.test('name:else'):
                result.else_ = self.parse_statements(('name:endif',),
                                                     drop_needle=True)
            break
        return result
1296657
private static List<Map<String, String>> loadWords(InputStream stream) {
    List<Map<String, String>> list = new ArrayList<>();
    try (
      InputStreamReader isr = new InputStreamReader(stream, FILE_ENCODING);
      BufferedReader br = new BufferedReader(isr);
    ) {
      String line;

      Tokenizer wordTokenizer = new Breton().getWordTokenizer();
      while ((line = br.readLine()) != null) {
        line = line.trim();
        if (line.isEmpty() || line.charAt(0) == '#') { // ignore comments
          continue;
        }
        String[] parts = line.split("=");
        if (parts.length != 2) {
          throw new IOException("Format error in file "
                  + JLanguageTool.getDataBroker().getFromRulesDirAsUrl(FILE_NAME)
                  + ", line: " + line);
        }
        String[] wrongForms = parts[0].split("\\|"); // multiple incorrect forms
        for (String wrongForm : wrongForms) {
          int wordCount = 0;
          List<String> tokens = wordTokenizer.tokenize(wrongForm);
          for (String token : tokens) {
            if (!StringTools.isWhitespace(token)) {
              wordCount++;
            }
          }
          // grow if necessary
          for (int i = list.size(); i < wordCount; i++) {
            list.add(new HashMap<>());
          }
          list.get(wordCount - 1).put(wrongForm, parts[1]);
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    // seal the result (prevent modification from outside this class)
    List<Map<String,String>> result = new ArrayList<>();
    for (Map<String, String> map : list) {
      result.add(Collections.unmodifiableMap(map));
    }
    return Collections.unmodifiableList(result);
  }