Remove unreachable symbols from the gramar
    :param grammar: Grammar where to symbols remove
    :param inplace: True if transformation should be performed in place. False by default.
    :return: Grammar without unreachable symbols.
764547
def remove_unreachable_symbols(grammar, inplace=False):
    # type: (Grammar, bool) -> Grammar
    
    # copy if required
    if inplace is False:
        grammar = copy(grammar)
    # check if start symbol is set
    if grammar.start is None:
        raise StartSymbolNotSetException()
    # create process sets
    reachable = {grammar.start}
    rules = grammar.rules.copy()
    # begin iterations
    while True:
        # create sets for current iteration
        active = reachable.copy()
        # loop the working rules
        for rule in rules.copy():
            # lf left part of rule already in reachable symbols
            if rule.fromSymbol in reachable:
                # set symbols on the right as reachable
                for symbol in rule.right:
                    active.add(symbol)
                # remove rule from the next iteration
                rules.remove(rule)
            # end of rules loop
        # if current and previous iterations are same, we are done
        if active == reachable:
            break
        # otherwise swap the sets
        reachable = active
    # remove the symbols
    nonterminals_to_remove = grammar.nonterminals.difference(reachable)
    terminals_to_remove = grammar.terminals.difference(reachable)
    grammar.nonterminals.remove(*nonterminals_to_remove)
    grammar.terminals.remove(*terminals_to_remove)
    # return grammar
    return grammar
1145076
def extract(grammar, inputdata, fixed_start = False, return_first=False):
    
    if not inputdata:
        return []
    checker = checker_factory(grammar)

    totallen = len(inputdata)
    from pydsl.grammar.PEG import Choice
    try:
        maxl = grammar.maxsize or totallen
    except NotImplementedError:
        maxl = totallen
    try:
        #minl = grammar.minsize #FIXME: It won't work with incompatible alphabets
        minl = 1
    except NotImplementedError:
        minl = 1
    if fixed_start:
        max_start = 1
    else:
        max_start = totallen
    result = []
    for i in range(max_start):
        for j in range(i+minl, min(i+maxl, totallen) + 1):
            slice = inputdata[i:j]
            check = checker.check(slice)
            if check:
                this_pt = PositionToken(slice, grammar, i, j)
                if return_first:
                    return this_pt
                result.append(this_pt)
    return result
1179191
def remove_punctuation(text, exceptions=[]):
    

    all_but = [
        r'\w',
        r'\s'
    ]

    all_but.extend(exceptions)

    pattern = '[^{}]'.format(''.join(all_but))

    return re.sub(pattern, '', text)
300364
def _parse(self, lines, grammar, ignore_comments=False):
        
        results = []
        for c in lines:
            if c != '' and not (ignore_comments and c[0] == '#'):
                try:
                    results.append(grammar.parseString(c))
                except pyparsing.ParseException as e:
                    raise ValueError('Invalid syntax. Verify line {} is '
                        'correct.\n{}\n\n{}'.format(e.lineno, c, e))
        return results
762574
def load_grammar(self, grammar_text, grammar_name='<?>'):
        ""

        try:
            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
        except UnexpectedCharacters as e:
            context = e.get_context(grammar_text)
            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
                               (e.line, e.column, grammar_name, context))
        except UnexpectedToken as e:
            context = e.get_context(grammar_text)
            error = e.match_examples(self.parser.parse, {
                'Unclosed parenthesis': ['a: (\n'],
                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
                'Alias expects lowercase name': ['a: -> "a"\n'],
                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
                '%import expects a name': ['%import "a"\n'],
                '%ignore expects a value': ['%ignore %import\n'],
            })
            if error:
                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
            elif 'STRING' in e.expected:
                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
            raise

        tree = PrepareGrammar().transform(tree)

        # Extract grammar items
        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
        term_defs = defs.pop('term', [])
        rule_defs = defs.pop('rule', [])
        statements = defs.pop('statement', [])
        assert not defs

        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
        rule_defs = [options_from_rule(*x) for x in rule_defs]

        # Execute statements
        ignore, imports = [], {}
        for (stmt,) in statements:
            if stmt.data == 'ignore':
                t ,= stmt.children
                ignore.append(t)
            elif stmt.data == 'import':
                if len(stmt.children) > 1:
                    path_node, arg1 = stmt.children
                else:
                    path_node, = stmt.children
                    arg1 = None

                if isinstance(arg1, Tree):  # Multi import
                    dotted_path = tuple(path_node.children)
                    names = arg1.children
                    aliases = dict(zip(names, names))  # Can't have aliased multi import, so all aliases will be the same as names
                else:  # Single import
                    dotted_path = tuple(path_node.children[:-1])
                    name = path_node.children[-1]  # Get name from dotted path
                    aliases = {name: arg1 or name}  # Aliases if exist

                if path_node.data == 'import_lib':  # Import from library
                    base_paths = []
                else:  # Relative import
                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
                        try:
                            base_file = os.path.abspath(sys.modules['__main__'].__file__)
                        except AttributeError:
                            base_file = None
                    else:
                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
                    if base_file:
                        base_paths = [os.path.split(base_file)[0]]
                    else:
                        base_paths = [os.path.abspath(os.path.curdir)]

                try:
                    import_base_paths, import_aliases = imports[dotted_path]
                    assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path)
                    import_aliases.update(aliases)
                except KeyError:
                    imports[dotted_path] = base_paths, aliases

            elif stmt.data == 'declare':
                for t in stmt.children:
                    term_defs.append([t.value, (None, None)])
            else:
                assert False, stmt

        # import grammars
        for dotted_path, (base_paths, aliases) in imports.items():
            grammar_path = os.path.join(*dotted_path) + EXT
            g = import_grammar(grammar_path, base_paths=base_paths)
            new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)

            term_defs += new_td
            rule_defs += new_rd

        # Verify correctness 1
        for name, _ in term_defs:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)

        # Handle ignore tokens
        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
        #     inability to handle duplicate terminals (two names, one value)
        ignore_names = []
        for t in ignore:
            if t.data=='expansions' and len(t.children) == 1:
                t2 ,= t.children
                if t2.data=='expansion' and len(t2.children) == 1:
                    item ,= t2.children
                    if item.data == 'value':
                        item ,= item.children
                        if isinstance(item, Token) and item.type == 'TERMINAL':
                            ignore_names.append(item.value)
                            continue

            name = '__IGNORE_%d'% len(ignore_names)
            ignore_names.append(name)
            term_defs.append((name, (t, 1)))

        # Verify correctness 2
        terminal_names = set()
        for name, _ in term_defs:
            if name in terminal_names:
                raise GrammarError("Terminal '%s' defined more than once" % name)
            terminal_names.add(name)

        if set(ignore_names) > terminal_names:
            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))

        resolve_term_references(term_defs)

        rules = rule_defs

        rule_names = set()
        for name, _x, _o in rules:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
            if name in rule_names:
                raise GrammarError("Rule '%s' defined more than once" % name)
            rule_names.add(name)

        for name, expansions, _o in rules:
            used_symbols = {t for x in expansions.find_data('expansion')
                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
            for sym in used_symbols:
                if sym.type == 'TERMINAL':
                    if sym not in terminal_names:
                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
                else:
                    if sym not in rule_names:
                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))

        return Grammar(rules, term_defs, ignore_names)
76952
def TermsProcessor(instance, placeholder, rendered_content, original_context):
    
    if 'terms' in original_context:
        return rendered_content

    return mark_safe(replace_terms(rendered_content))
701066
def transform_to_chomsky_normal_form(grammar, inplace=False):
    # type: (Grammar, bool) -> Grammar
    
    # Copy if required
    if inplace is False:
        grammar = copy(grammar)
    # Create nonterminals rewritable to the terminal.
    # They will be inserted into the grammar as needed.
    fill = TerminalsFilling(grammar)
    to_process = Queue()
    for r in grammar.rules:
        to_process.put(r)
    while not to_process.empty():
        rule = to_process.get()  # type: Type[Rule]
        # Check, if rule must be split
        if len(rule.right) > 2:
            grammar.rules.remove(rule)
            # create nonterm that represent group on the right
            created_nonterm = type("ChomskyGroup[" + rule.__name__ + "]",
                                   (ChomskyGroupNonterminal,),
                                   ChomskyGroupNonterminal.__dict__.copy())  # type: Type[ChomskyGroupNonterminal]
            created_nonterm.group = rule.right[1:]
            # create rule that replace current
            created_left_rule = type("ChomskySplit[" + rule.__name__ + "]",
                                     (ChomskySplitRule,),
                                     ChomskySplitRule.__dict__.copy())  # type: Type[ChomskySplitRule]
            created_left_rule.rule = ([rule.fromSymbol], [rule.right[0], created_nonterm])
            created_left_rule.from_rule = rule
            # create rule with symbols on the right
            created_right_rule = type("ChomskyRest[" + rule.__name__ + "]",
                                      (ChomskyRestRule,),
                                      ChomskyRestRule.__dict__.copy())  # type: Type[ChomskyRestRule]
            created_right_rule.rule = ([created_nonterm], rule.right[1:])
            created_right_rule.from_rule = rule
            # add it to the grammar
            grammar.nonterminals.add(created_nonterm)
            grammar.rules.add(created_left_rule, created_right_rule)
            to_process.put(created_left_rule)
            to_process.put(created_right_rule)
        # check, if must replace terminal
        elif len(rule.right) == 2:
            if rule.right[0] in grammar.terminals:
                # first symbol is terminal
                # remove rule from grammar
                grammar.rules.remove(rule)
                # get nonterminal rewritable to the terminal and add that rules into the grammar implicitly
                symb = fill.get(rule.right[0])
                # create rule replacing the original one
                created = type("ChomskyLeft[" + rule.__name__ + "]",
                               (ChomskyTerminalReplaceRule,),
                               ChomskyTerminalReplaceRule.__dict__.copy())  # type: Type[ChomskyTerminalReplaceRule]
                created.rule = ([rule.fromSymbol], [symb, rule.right[1]])
                created.from_rule = rule
                created.replaced_index = 0
                # add it into the grammar
                grammar.rules.add(created)
                to_process.put(created)
            elif rule.right[1] in grammar.terminals:
                # second symbol is terminal
                # remove rule from grammar
                grammar.rules.remove(rule)
                # get nonterminal rewritable to the terminal and add that rules into the grammar implicitly
                symb = fill.get(rule.right[1])
                # create rule replacing the original one
                created = type("ChomskyRight[" + rule.__name__ + "]",
                               (ChomskyTerminalReplaceRule,),
                               ChomskyTerminalReplaceRule.__dict__.copy())
                created.rule = ([rule.fromSymbol], [rule.right[0], symb])
                created.from_rule = rule
                created.replaced_index = 1
                # add it into the grammar
                grammar.rules.add(created)
                to_process.put(created)
    return grammar
10073
def process_grammars(self, grammars):
        """"""
        for path in self.default_grammars:
            grammars.insert(0, open(os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                                                 os.path.normcase(path)))))
        for fo in grammars:
            logging.debug("Processing grammar content of %s", fo.name)
            self.set_namespace(os.path.splitext(os.path.basename(fo.name))[0])
            for line in fo:
                self.parse_line(line)
            self.handle_empty_line()
        self.resolve_xref()
        self.calculate_leaf_paths()
633539
function undeclaredGrammar(grammarName, namespace, interval) {
  var message = namespace ?
      'Grammar ' + grammarName + ' is not declared in namespace ' + Namespace.toString(namespace) :
      ' ' + grammarName;
  return createError(message, interval);
}
504125
function removeInvalidNodeErrorsInIdentifierOrLiteral(node) {
            const shouldCheckStrings = skipStrings && (typeof node.value === "string");
            const shouldCheckRegExps = skipRegExps && Boolean(node.regex);

            if (shouldCheckStrings || shouldCheckRegExps) {

                // If we have irregular characters remove them from the errors list
                if (ALL_IRREGULARS.test(node.raw)) {
                    removeWhitespaceError(node);
                }
            }
        }
969869
def remove_useless(self):
        """"""
        if not self.is_contextfree():
            raise ValueError("grammar must be context-free")
        by_lhs = collections.defaultdict(list)
        by_rhs = collections.defaultdict(list)
        for [lhs], rhs in self.rules:
            by_lhs[lhs].append((lhs, rhs))
            for y in rhs:
                if y in self.nonterminals:
                    by_rhs[y].append((lhs, rhs))
            
        agenda = collections.deque([self.start])
        reachable = set()
        while len(agenda) > 0:
            x = agenda.popleft()
            if x in reachable: continue
            reachable.add(x)
            for _, rhs in by_lhs[x]:
                for y in rhs:
                    if y in by_lhs:
                        agenda.append(y)

        agenda = collections.deque()
        productive = set()
        for [lhs], rhs in self.rules:
            if all(y not in self.nonterminals for y in rhs):
                agenda.append(lhs)
        while len(agenda) > 0:
            y = agenda.popleft()
            if y in productive: continue
            productive.add(y)
            for lhs, rhs in by_rhs[y]:
                if all(y not in self.nonterminals or y in productive for y in rhs):
                    agenda.append(lhs)

        g = Grammar()
        g.set_start(self.start)

        for [lhs], rhs in self.rules:
            if (lhs in reachable & productive and
                all(y not in self.nonterminals or y in reachable & productive for y in rhs)):
                g.add_rule([lhs], rhs)
        return g
92564
def match_in(grammar, text):
    """"""
    for result in grammar.parseWithTabs().scanString(text):
        return True
    return False
622180
private static <E> boolean removeOccurrencesImpl(
      Multiset<E> multisetToModify, Multiset<?> occurrencesToRemove) {
    // TODO(user): generalize to removing an Iterable, perhaps
    checkNotNull(multisetToModify);
    checkNotNull(occurrencesToRemove);

    boolean changed = false;
    Iterator<Entry<E>> entryIterator = multisetToModify.entrySet().iterator();
    while (entryIterator.hasNext()) {
      Entry<E> entry = entryIterator.next();
      int removeCount = occurrencesToRemove.count(entry.getElement());
      if (removeCount >= entry.getCount()) {
        entryIterator.remove();
        changed = true;
      } else if (removeCount > 0) {
        multisetToModify.remove(entry.getElement(), removeCount);
        changed = true;
      }
    }
    return changed;
  }
1047819
def accept_operator(self, precedence):
        """"""
        match = grammar.infix(self.tokens)
        if not match:
            return

        if match.operator.precedence < precedence:
            return

        # The next thing is an operator that we want. Now match it for real.
        return self.tokens.accept(grammar.infix)
273725
function removeInvalidNodeErrorsInIdentifierOrLiteral(node) {
			var shouldCheckStrings = skipStrings && (typeof node.value === "string");
			var shouldCheckRegExps = skipRegExps && (node.value instanceof RegExp);

			if (shouldCheckStrings || shouldCheckRegExps) {

				// If we have irregular characters remove them from the errors list
				if (ALL_IRREGULARS.test(node.raw)) {
					removeWhitespaceError(node);
				}
			}
		}
803955
def _extract_alphabet(self, grammar):
        
        alphabet = set([])
        for terminal in grammar.Terminals:
            alphabet |= set([x for x in terminal])
        self.alphabet = list(alphabet)
1289104
function(env, language) {
          if (env.language !== language || !env.tokenStack) {
            return
          }
          // Switch the grammar back
          env.grammar = Prism.languages[language]
          var j = 0
          var keys = Object.keys(env.tokenStack)
          function walkTokens(tokens) {
            for (var i = 0; i < tokens.length; i++) {
              // all placeholders are replaced already
              if (j >= keys.length) {
                break
              }
              var token = tokens[i]
              if (
                typeof token === 'string' ||
                (token.content && typeof token.content === 'string')
              ) {
                var k = keys[j]
                var t = env.tokenStack[k]
                var s = typeof token === 'string' ? token : token.content
                var placeholder = getPlaceholder(language, k)
                var index = s.indexOf(placeholder)
                if (index > -1) {
                  ++j
                  var before = s.substring(0, index)
                  var middle = new Prism.Token(
                    language,
                    Prism.tokenize(t, env.grammar),
                    'language-' + language,
                    t
                  )
                  var after = s.substring(index + placeholder.length)
                  var replacement = []
                  if (before) {
                    replacement.push.apply(replacement, walkTokens([before]))
                  }
                  replacement.push(middle)
                  if (after) {
                    replacement.push.apply(replacement, walkTokens([after]))
                  }
                  if (typeof token === 'string') {
                    tokens.splice.apply(tokens, [i, 1].concat(replacement))
                  } else {
                    token.content = replacement
                  }
                }
              } else if (
                token.content /* && typeof token.content !== 'string' */
              ) {
                walkTokens(token.content)
              }
            }
            return tokens
          }
          walkTokens(env.tokens)
        }