Process provided grammars by parsing them into Python objects.
10073
def process_grammars(self, grammars):
        """"""
        for path in self.default_grammars:
            grammars.insert(0, open(os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                                                 os.path.normcase(path)))))
        for fo in grammars:
            logging.debug("Processing grammar content of %s", fo.name)
            self.set_namespace(os.path.splitext(os.path.basename(fo.name))[0])
            for line in fo:
                self.parse_line(line)
            self.handle_empty_line()
        self.resolve_xref()
        self.calculate_leaf_paths()
602935
def _process(self, input):
        '''
        
        '''

        input = re.sub("<[^>]*>", " ", input) 
        punct = list(string.punctuation)
        for symbol in punct:
            input = input.replace(symbol, " %s " % symbol)
        input = filter(lambda x: x != u'', input.lower().split(' '))
        return input
433684
def parse(cls, parser, text, pos):
        """"""
        try:
            parser._parsing_parenthesized_simple_values_expression = True
            remaining_text, recognized_tokens = parser.parse(text, cls.grammar)
            return remaining_text, recognized_tokens
        except SyntaxError as e:
            return text, e
        finally:
            parser._parsing_parenthesized_simple_values_expression = False
319795
def parse(tokens):
    ''''''
    args = []
    kwargs = {}
    last = None
    for token in tokens:
        if token.startswith('--'):
            # If this is a keyword flag, but we've already got one that we've
            # parsed, then we're going to interpret it as a bool
            if last:
                kwargs[last] = True
            # See if it is the --foo=5 style
            last, _, value = token.strip('-').partition('=')
            if value:
                kwargs[last] = value
                last = None
        elif last != None:
            kwargs[last] = token
            last = None
        else:
            args.append(token)

    # If there's a dangling last, set that bool
    if last:
        kwargs[last] = True

    return args, kwargs
762574
def load_grammar(self, grammar_text, grammar_name='<?>'):
        ""

        try:
            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
        except UnexpectedCharacters as e:
            context = e.get_context(grammar_text)
            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
                               (e.line, e.column, grammar_name, context))
        except UnexpectedToken as e:
            context = e.get_context(grammar_text)
            error = e.match_examples(self.parser.parse, {
                'Unclosed parenthesis': ['a: (\n'],
                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
                'Alias expects lowercase name': ['a: -> "a"\n'],
                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
                '%import expects a name': ['%import "a"\n'],
                '%ignore expects a value': ['%ignore %import\n'],
            })
            if error:
                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
            elif 'STRING' in e.expected:
                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
            raise

        tree = PrepareGrammar().transform(tree)

        # Extract grammar items
        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
        term_defs = defs.pop('term', [])
        rule_defs = defs.pop('rule', [])
        statements = defs.pop('statement', [])
        assert not defs

        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
        rule_defs = [options_from_rule(*x) for x in rule_defs]

        # Execute statements
        ignore, imports = [], {}
        for (stmt,) in statements:
            if stmt.data == 'ignore':
                t ,= stmt.children
                ignore.append(t)
            elif stmt.data == 'import':
                if len(stmt.children) > 1:
                    path_node, arg1 = stmt.children
                else:
                    path_node, = stmt.children
                    arg1 = None

                if isinstance(arg1, Tree):  # Multi import
                    dotted_path = tuple(path_node.children)
                    names = arg1.children
                    aliases = dict(zip(names, names))  # Can't have aliased multi import, so all aliases will be the same as names
                else:  # Single import
                    dotted_path = tuple(path_node.children[:-1])
                    name = path_node.children[-1]  # Get name from dotted path
                    aliases = {name: arg1 or name}  # Aliases if exist

                if path_node.data == 'import_lib':  # Import from library
                    base_paths = []
                else:  # Relative import
                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
                        try:
                            base_file = os.path.abspath(sys.modules['__main__'].__file__)
                        except AttributeError:
                            base_file = None
                    else:
                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
                    if base_file:
                        base_paths = [os.path.split(base_file)[0]]
                    else:
                        base_paths = [os.path.abspath(os.path.curdir)]

                try:
                    import_base_paths, import_aliases = imports[dotted_path]
                    assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path)
                    import_aliases.update(aliases)
                except KeyError:
                    imports[dotted_path] = base_paths, aliases

            elif stmt.data == 'declare':
                for t in stmt.children:
                    term_defs.append([t.value, (None, None)])
            else:
                assert False, stmt

        # import grammars
        for dotted_path, (base_paths, aliases) in imports.items():
            grammar_path = os.path.join(*dotted_path) + EXT
            g = import_grammar(grammar_path, base_paths=base_paths)
            new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)

            term_defs += new_td
            rule_defs += new_rd

        # Verify correctness 1
        for name, _ in term_defs:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)

        # Handle ignore tokens
        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
        #     inability to handle duplicate terminals (two names, one value)
        ignore_names = []
        for t in ignore:
            if t.data=='expansions' and len(t.children) == 1:
                t2 ,= t.children
                if t2.data=='expansion' and len(t2.children) == 1:
                    item ,= t2.children
                    if item.data == 'value':
                        item ,= item.children
                        if isinstance(item, Token) and item.type == 'TERMINAL':
                            ignore_names.append(item.value)
                            continue

            name = '__IGNORE_%d'% len(ignore_names)
            ignore_names.append(name)
            term_defs.append((name, (t, 1)))

        # Verify correctness 2
        terminal_names = set()
        for name, _ in term_defs:
            if name in terminal_names:
                raise GrammarError("Terminal '%s' defined more than once" % name)
            terminal_names.add(name)

        if set(ignore_names) > terminal_names:
            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))

        resolve_term_references(term_defs)

        rules = rule_defs

        rule_names = set()
        for name, _x, _o in rules:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
            if name in rule_names:
                raise GrammarError("Rule '%s' defined more than once" % name)
            rule_names.add(name)

        for name, expansions, _o in rules:
            used_symbols = {t for x in expansions.find_data('expansion')
                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
            for sym in used_symbols:
                if sym.type == 'TERMINAL':
                    if sym not in terminal_names:
                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
                else:
                    if sym not in rule_names:
                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))

        return Grammar(rules, term_defs, ignore_names)
79343
def _parse_programs(self, string, parent, filepath=None):
        """"""
        #First, get hold of the docstrings  for all the modules so that we can
        #attach them as we parse them.
        moddocs = self.docparser.parse_docs(string)
        #Now look for modules in the file and then match them to their decorators.
        matches = self.RE_PROGRAM.finditer(string)
        result = []
        for rmodule in matches:
            name = rmodule.group("name").lower()
            contents = re.sub("&[ ]*\n", "", rmodule.group("contents"))
            module = self._process_module(name, contents, parent, rmodule, filepath)
            #Check whether the docparser found docstrings for the module.
            if name in moddocs:                
                module.docstring = self.docparser.to_doc(moddocs[name][0], name)
                module.docstart, module.docend = module.absolute_charindex(string, moddocs[name][1],
                                                                           moddocs[name][2])
            result.append(module)
        return result
789232
private void convertSingleProductions(String productionName,
	    ParseTreeNode productionConstructions) throws TreeException,
	    GrammarException {
	for (ParseTreeNode productionConstruction : productionConstructions
		.getChildren("ProductionConstruction")) {
	    convertSingleProduction(productionName, productionConstruction);
	}
    }
997923
def dump_grammar(self, out=sys.stdout):
        
        for rule in sorted(self.rule2name.items()):
            out.write("%s\n" % rule2str(rule[0]))
        return
93282
def parse(self, tokenized):  # pylint: disable=invalid-name
        """"""
        table, trees = _parse(tokenized, self.grammar)
        # Check if the parse succeeded.
        if all(r.lhs != self.start for r in table[(0, len(tokenized) - 1)]):
            raise ParseError('Parsing failed.')
        parse = trees[(0, len(tokenized) - 1)][self.start]
        return self._to_tree(revert_cnf(parse))
345848
def make_grammar(allow_errors):
  """"""
  if allow_errors in GRAMMAR_CACHE:
    return GRAMMAR_CACHE[allow_errors]

  tuple = p.Forward()
  catch_errors = p.Forward()
  catch_errors << (p.Regex('[^{};]*') - p.Optional(tuple) - p.Regex('[^;}]*'))

  def swallow_remainder():
    if allow_errors:
      return pattern('swallow_remainder', p.Suppress(catch_errors))
    return p.Empty()

  def swallow_errors(rule):
    """Extend the production rule by potentially eating errors.

    This does not return a p.NoMatch() because that messes up the error messages.
    """
    ret = rule
    if allow_errors:
      # Synchronize on the first semicolon or the first unbalanced closing curly
      ret = rule | pattern('catch_errors', parseWithLocation(p.Suppress(catch_errors), UnparseableNode))
    return ret

  class Grammar:
    keywords = ['and', 'or', 'not', 'if', 'then', 'else', 'include', 'inherit', 'null', 'true', 'false',
        'for', 'in']

    # This is a hack: this condition helps uselessly recursing into the grammar for
    # juxtapositions.
    early_abort_scan = ~p.oneOf([';', ',', ']', '}', 'for' ])

    expression = pattern('expression', p.Forward())

    comment = p.Regex('#') + ~p.FollowedBy(sym('.')) + p.restOfLine
    doc_comment = pattern('doc_comment', (sym('#.') - p.restOfLine))

    quotedIdentifier = pattern('quotedIdentifier', p.QuotedString('`', multiline=False))

    # - Must start with an alphascore
    # - May contain alphanumericscores and special characters such as : and -
    # - Must not end in a special character
    identifier = pattern('identifier', parseWithLocation(quotedIdentifier | p.Regex(r'[a-zA-Z_]([a-zA-Z0-9_:-]*[a-zA-Z0-9_])?'), Identifier))

    # Variable identifier (can't be any of the keywords, which may have lower matching priority)
    variable = pattern('variable', ~p.MatchFirst(p.oneOf(keywords)) + pattern('identifier', parseWithLocation(identifier.copy(), Var)))

    # Contants
    integer = pattern('integer', parseWithLocation(p.Word(p.nums), convertAndMake(int, Literal)))
    floating = pattern('floating', parseWithLocation(p.Regex(r'\d*\.\d+'), convertAndMake(float, Literal)))
    dq_string = pattern('dq_string', parseWithLocation(p.QuotedString('"', escChar='\\', unquoteResults=False, multiline=True), convertAndMake(unquote, Literal)))
    sq_string = pattern('sq_string', parseWithLocation(p.QuotedString("'", escChar='\\', unquoteResults=False, multiline=True), convertAndMake(unquote, Literal)))
    boolean = pattern('boolean', parseWithLocation(p.Keyword('true') | p.Keyword('false'), convertAndMake(mkBool, Literal)))
    null = pattern('null', parseWithLocation(p.Keyword('null'), Null))

    # List
    list_ = pattern('list', parseWithLocation(bracketedList('[', ']', ',', expression), List))

    # Tuple
    inherit = pattern('inherit', (kw('inherit') - p.ZeroOrMore(variable)).setParseAction(inheritNodes))
    schema_spec = pattern('schema_spec', parseWithLocation(p.Optional(p.Keyword('private').setParseAction(lambda: True), default=False)
                  - p.Optional(p.Keyword('required').setParseAction(lambda: True), default=False)
                  - p.Optional(expression, default=any_schema_expr), MemberSchemaNode))
    optional_schema = pattern('optional_schema', p.Optional(p.Suppress(':') - schema_spec, default=no_schema))

    expression_value = pattern('expression_value', sym('=') - swallow_errors(expression))
    void_value = pattern('void_value', parseWithLocation(p.FollowedBy(sym(';') | sym('}')), lambda loc: Void(loc, 'nonameyet')))
    member_value = pattern('member_value', swallow_errors(expression_value | void_value))
    named_member = pattern('named_member', parseWithLocation(identifier - optional_schema - member_value - swallow_remainder(), TupleMemberNode))
    documented_member = pattern('documented_member', parseWithLocation(parseWithLocation(p.ZeroOrMore(doc_comment), DocComment) + named_member, attach_doc_comment))
    tuple_member = early_abort_scan + pattern('tuple_member', swallow_errors(inherit | documented_member) - swallow_remainder())

    ErrorAwareTupleNode = functools.partial(TupleNode, allow_errors)
    tuple_members = pattern('tuple_members', parseWithLocation(listMembers(';', tuple_member), ErrorAwareTupleNode))
    tuple << pattern('tuple', parseWithLocation(bracketedList('{', '}', ';', tuple_member, allow_missing_close=allow_errors), ErrorAwareTupleNode))

    # Argument list will live by itself as a atom. Actually, it's a tuple, but we
    # don't call it that because we use that term for something else already :)
    arg_list = pattern('arg_list', bracketedList('(', ')', ',', expression).setParseAction(ArgList))

    parenthesized_expr = pattern('parenthesized_expr', (sym('(') - expression - ')').setParseAction(head))

    unary_op = pattern('unary_op', (p.oneOf(' '.join(functions.unary_operators.keys())) - expression).setParseAction(mkUnOp))

    if_then_else = pattern('if_then_else', parseWithLocation(kw('if') + expression +
                    kw('then') + expression +
                    kw('else') + expression, Condition))

    list_comprehension = pattern('list_comprehension', parseWithLocation(sym('[') + expression + kw('for') + variable + kw('in') +
        expression + p.Optional(kw('if') + expression) + sym(']'), ListComprehension))


    # We don't allow space-application here
    # Now our grammar is becoming very dirty and hackish
    deref = pattern('deref', p.Forward())
    include = pattern('include', parseWithLocation(kw('include') - deref, Include))

    atom = pattern('atom', (tuple
            | sq_string
            | dq_string
            | variable
            | floating
            | integer
            | boolean
            | list_
            | null
            | unary_op
            | parenthesized_expr
            | if_then_else
            | include
            | list_comprehension
            ))

    # We have two different forms of function application, so they can have 2
    # different precedences. This one: fn(args), which binds stronger than
    # dereferencing (fn(args).attr == (fn(args)).attr)
    applic1 = pattern('applic1', parseWithLocation(atom - p.ZeroOrMore(arg_list), mkApplications))

    # Dereferencing of an expression (obj.bar)
    deref << parseWithLocation(applic1 - p.ZeroOrMore(p.Suppress('.') - swallow_errors(identifier)), mkDerefs)

    # All binary operators at various precedence levels go here:
    # This piece of code does the moral equivalent of:
    #
    #     T = F*F | F/F | F
    #     E = T+T | T-T | T
    #
    # etc.
    term = deref
    for op_level in functions.binary_operators_before_juxtaposition:
      operator_syms = list(op_level.keys())
      term = (term - p.ZeroOrMore(p.oneOf(operator_syms) - term)).setParseAction(mkBinOps)

    # Juxtaposition function application (fn arg), must be 1-arg every time
    applic2 = pattern('applic2', parseWithLocation(term - p.ZeroOrMore(early_abort_scan + term), mkApplications))

    term = applic2
    for op_level in functions.binary_operators_after_juxtaposition:
      operator_syms = list(op_level.keys())
      term = (term - p.ZeroOrMore(p.oneOf(operator_syms) - term)).setParseAction(mkBinOps)

    expression << term

    # Two entry points: start at an arbitrary expression, or expect the top-level
    # scope to be a tuple.
    start = pattern('start', expression.copy().ignore(comment))
    start_tuple = tuple_members.ignore(comment)
  GRAMMAR_CACHE[allow_errors] = Grammar
  return Grammar
1108750
def from_dp(cls, text, use_unicode_for_glyphs=True):
        
        from .grammars.dp import parse as parse_dp
        if isinstance(text, bytes):
            original_bytes = text
            text = text.decode('utf8')
        else:
            original_bytes = text.encode('utf8')
        instance = parse_dp(text, use_unicode_for_glyphs=use_unicode_for_glyphs)
        instance.original_type = 'dp'
        instance.original_bytes = original_bytes
        return instance
340041
def _get_parser_compound(cls, *parse_methods):
        
        def parse(value):
            parsed = value

            for method in parse_methods:
                parsed = method(parsed)

            return parsed

        return parse
579410
def _parse(self, parsable, parser_name):
        '''
        
        '''
        calc, error = None, None
        try:
            for calc in self.Parsers[parser_name].iparse(parsable):
                yield calc, None
            return
        except RuntimeError as e:
            error = "routine %s parser error in %s: %s" % ( parser_name, parsable, e )
        except:
            exc_type, exc_value, exc_tb = sys.exc_info()
            error = "unexpected %s parser error in %s:\n %s" % ( parser_name, parsable, "".join(traceback.format_exception( exc_type, exc_value, exc_tb )) )
        yield None, error
1345458
def match_tokens(expected_tokens):
    """"""
    if isinstance(expected_tokens, Token):
        # Match a single token.
        def _grammar_func(tokens):
            try:
                next_token = next(iter(tokens))
            except StopIteration:
                return

            if next_token == expected_tokens:
                return TokenMatch(None, next_token.value, (next_token,))

    elif isinstance(expected_tokens, tuple):
        # Match multiple tokens.
        match_len = len(expected_tokens)
        def _grammar_func(tokens):
            upcoming = tuple(itertools.islice(tokens, match_len))
            if upcoming == expected_tokens:
                return TokenMatch(None, None, upcoming)
    else:
        raise TypeError(
            "'expected_tokens' must be an instance of Token or a tuple "
            "thereof. Got %r." % expected_tokens)

    return _grammar_func
1198905
def pattern_convert(grammar, raw_node_info):
    """"""
    type, value, context, children = raw_node_info
    if children or type in grammar.number2symbol:
        return pytree.Node(type, children, context=context)
    else:
        return pytree.Leaf(type, value, context=context)
992902
def ParseObjs(self, objs, expression):
    """"""
    filt = self._Compile(expression)
    for result in filt.Filter(objs):
      yield result
222088
def process(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    data(dataset, **ctx.obj).process(**kwargs)