Customized format and print for our kind of tokens
        which gets called in debugging grammar reduce rules
351166
def debug_reduce(self, rule, tokens, parent, i):
        
        prefix = '           '
        if parent and tokens:
            p_token = tokens[parent]
            if hasattr(p_token, 'line'):
                prefix = 'L.%3d.%03d: ' % (p_token.line, p_token.column)
                pass
            pass
        print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
552782
function generateSubstitution(token, substitutionPattern, substitutions) {
  var substitution = substitutionPattern.replace(/{([^}]+)}/ig, (all, group)=> {
    switch (group) {
      case 'namespace':
        return token.namespace;
      case 'name':
        return token.className;
      case 'random':
        return 'r' + Math.random().toString(36).substring(7);
      case 'hash':
        return 'h' + (token.hash + 0.0).toString(36);
      case 'unique':
        return 'u' + (++uniqueCounter);
    }
  });

  substitutions[token.className] = [];
  substitutions[token.className].push(substitution);
  return substitution;
}
1289939
def emptyPrinter(self, expr):
        """"""
        indent_str = "    " * (self._print_level - 1)
        lines = []
        if isinstance(expr.__class__, Singleton):
            # We exploit that Singletons override __expr__ to directly return
            # their name
            return indent_str + repr(expr)
        if isinstance(expr, Expression):
            args = expr.args
            keys = expr.minimal_kwargs.keys()
            lines.append(indent_str + expr.__class__.__name__ + "(")
            for arg in args:
                lines.append(self.doprint(arg) + ",")
            for key in keys:
                arg = expr.kwargs[key]
                lines.append(
                    ("    " * self._print_level) + key + '=' +
                    self.doprint(arg).lstrip() + ",")
            if len(args) > 0 or len(keys) > 0:
                lines[-1] = lines[-1][:-1]  # drop trailing comma for last arg
            lines[-1] += ")"
        elif isinstance(expr, (tuple, list)):
            delims = ("(", ")") if isinstance(expr, tuple) else ("[", "]")
            if len(expr) == 1:
                delims = (delims[0], "," + delims[1])
            lines.append(
                indent_str + delims[0] +
                ", ".join([render_head_repr(v) for v in expr]) +
                delims[1])
        else:
            lines.append(indent_str + SympyReprPrinter().doprint(expr))
        return "\n".join(lines)
1008415
private static String print(final String indent, JDToken token) {
        return token.match(new JDToken.MatchBlock<String>() {

            @Override
            public String _case(JDAsterisk x) {
                return "*";
            }

            @Override
            public String _case(JDEOL x) {
                return x.content + indent;
            }

            @Override
            public String _case(JDTag x) {
                return x.name;
            }

            @Override
            public String _case(JDWord x) {
                return x.word;
            }

            @Override
            public String _case(JDWhiteSpace x) {
                return x.ws;
            }
        });
    }
1055211
def _compileRegExp(string, insensitive, minimal):
        
        flags = 0
        if insensitive:
            flags = re.IGNORECASE

        string = string.replace('[_[:alnum:]]', '[\\w\\d]') # ad-hoc fix for C++ parser
        string = string.replace('[:digit:]', '\\d')
        string = string.replace('[:blank:]', '\\s')

        try:
            return re.compile(string, flags)
        except (re.error, AssertionError) as ex:
            _logger.warning("Invalid pattern '%s': %s", string, str(ex))
            return None
984004
def _simplify(self, pattern):
        # type: (str) -> str
        
        """
        # remove optional params
        # TODO(dcramer): it'd be nice to change these into [%s] but it currently
        # conflicts with the other rules because we're doing regexp matches
        # rather than parsing tokens
        result = self._optional_group_matcher.sub(lambda m: "%s" % m.group(1), pattern)

        # handle named groups first
        result = self._named_group_matcher.sub(lambda m: "{%s}" % m.group(1), result)

        # handle non-named groups
        result = self._non_named_group_matcher.sub("{var}", result)

        # handle optional params
        result = self._either_option_matcher.sub(lambda m: m.group(1), result)

        # clean up any outstanding regex-y characters.
        result = (
            result.replace("^", "")
            .replace("$", "")
            .replace("?", "")
            .replace("//", "/")
            .replace("\\", "")
        )

        return result
655891
def line_print(self):
        
        inv_types = {v: k for k, v in defines.Types.items()}

        if self._code is None:
            self._code = defines.Codes.EMPTY.number

        msg = "From {source}, To {destination}, {type}-{mid}, {code}-{token}, ["\
            .format(source=self._source, destination=self._destination, type=inv_types[self._type], mid=self._mid,
                    code=defines.Codes.LIST[self._code].name, token=self._token)
        for opt in self._options:
            msg += "{name}: {value}, ".format(name=opt.name, value=opt.value)
        msg += "]"
        if self.payload is not None:
            if isinstance(self.payload, dict):
                tmp = list(self.payload.values())[0][0:20]
            else:
                tmp = self.payload[0:20]
            msg += " {payload}...{length} bytes".format(payload=tmp, length=len(self.payload))
        else:
            msg += " No payload"
        return msg
1305086
def format_grammar_string(grammar_dictionary: Dict[str, List[str]]) -> str:
    
    grammar_string = '\n'.join([f"{nonterminal} = {' / '.join(right_hand_side)}"
                                for nonterminal, right_hand_side in grammar_dictionary.items()])
    return grammar_string.replace("\\", "\\\\")
1006206
def make_tokenizer(specs):
    """"""

    def compile_spec(spec):
        name, args = spec
        return name, re.compile(*args)

    compiled = [compile_spec(s) for s in specs]

    def match_specs(specs, str, i, position):
        line, pos = position
        for type, regexp in specs:
            m = regexp.match(str, i)
            if m is not None:
                value = m.group()
                nls = value.count(u'\n')
                n_line = line + nls
                if nls == 0:
                    n_pos = pos + len(value)
                else:
                    n_pos = len(value) - value.rfind(u'\n') - 1
                return Token(type, value, (line, pos + 1), (n_line, n_pos))
        else:
            errline = str.splitlines()[line - 1]
            raise LexerError((line, pos + 1), errline)

    def f(str):
        length = len(str)
        line, pos = 1, 0
        i = 0
        while i < length:
            t = match_specs(compiled, str, i, (line, pos))
            yield t
            line, pos = t.end
            i += len(t.value)

    return f
661190
def parse_plain_scalar_indent(TokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if len(text) <= context.indent:
            context.stack.pop()
            context.stack.pop()
            return
        if text:
            yield match.start(), TokenClass, text
            context.pos = match.end()
    return callback
916685
def _print mode, text, opts, caler, bind, cfg
      log_ = opts&NOLOG==0
      print_ = opts&NOPRINT==0
      str = cfg.fmt.dup
      str.gsub! "%mode", @highlight[mode]
      if bind
        text = bind.report text
      elsif !text.is String
        text = text.inspect
      elsif cfg.detect_comments and text =~ /\A[ \t]*#[ \t]+\S/
        text = "\n" + @c.green(text.gsub(/^([ \t]*#[ \t])?/, cfg.precede_comments).chop)
      end
      out = cfg.out
      if cfg._time or cfg.path_format
        now = Time.now
        if cfg._time
          time = now.strftime cfg.tf[0]
          time << ".#{cfg.tf[1]%[now.usec/1000]}" if cfg.tf[1]
          str.gsub! "%time", time
        end
        out = now.strftime cfg.out if cfg.path_format
      end
      if caler
        caler.sub!(/block (?:\((\d+) levels\) )?in/) {"{#{$1||1}}"}
        str.gsub! "%caller", caler.sub(String::SIMPLE_CALLER_RE, cfg.cf)
      end
      str.gsub! "%text", text
      str << "\n" if opts&INLINE==0
      log_str = cfg.color_out ? str : @c.clean(str)
      RMTools.write out, log_str if log_
      Kernel::print str if print_
    end
334190
function initializeParser(document, debug) {
    const chars = new antlr.InputStream(document);
    const lexer = new grammar.DocumentLexer(chars);
    const listener = new CustomErrorListener(debug);
    lexer.removeErrorListeners();
    lexer.addErrorListener(listener);
    const tokens = new antlr.CommonTokenStream(lexer);
    const parser = new grammar.DocumentParser(tokens);
    parser.buildParseTrees = true;
    parser.removeErrorListeners();
    parser.addErrorListener(listener);
    parser._errHandler = new CustomErrorStrategy();
    return parser;
}
1218848
def _mpda(self, re_grammar, splitstring=0):
        
        cnfgrammar = CNFGenerator(re_grammar)

        if not self.alphabet:
            self._extract_alphabet(cnfgrammar)

        cnftopda = CnfPda(self.alphabet)
        productions = {}
        nonterminals = []
        nonterminals.append(cnfgrammar.init_symbol)
        for key in list(cnfgrammar.grammar_nonterminals):
            if key != cnfgrammar.init_symbol:
                nonterminals.append(key)
        for key in list(cnfgrammar.grammar_nonterminals):
            j = 0
            productions[key] = {}
            # print 'testing '+key
            for pair in cnfgrammar.grammar_rules:
                cnf_form = list(pair)
                if cnf_form[0] == key:
                    productions[key][j] = {}
                    if isinstance(cnf_form[1], type(())):
                        #            print list(p[1])
                        productions[key][j]['b0'] = list(cnf_form[1])[0]
                        productions[key][j]['b1'] = list(cnf_form[1])[1]
                    else:
                        #           print p[1]
                        productions[key][j]['a'] = cnf_form[1]
                    j = j + 1
        return cnftopda.initialize(
            nonterminals, productions, list(
                cnfgrammar.grammar_terminals), splitstring)
1148176
def format_python2_stmts(python_stmts, show_tokens=False, showast=False,
                         showgrammar=False, compile_mode='exec'):
    

    parser_debug = {'rules': False, 'transition': False,
                    'reduce': showgrammar,
                    'errorstack': True, 'context': True, 'dups': True }
    parsed = parse_python2(python_stmts, show_tokens=show_tokens,
                           parser_debug=parser_debug)
    assert parsed == 'file_input', 'Should have parsed grammar start'

    formatter = Python2Formatter()

    if showast:
        print(parsed)

    # What we've been waiting for: Generate source from AST!
    python2_formatted_str = formatter.traverse(parsed)

    return python2_formatted_str
474641
private function printActiveFormattingElements() {
        if (!$this->a_formatting) {
            return;
        }
        $names = array();
        foreach ($this->a_formatting as $node) {
            if ($node === self::MARKER) {
                $names[] = 'MARKER';
            } else {
                $names[] = $node->tagName;
            }
        }
        echo "  -> active formatting [" . implode(', ', $names) . "]\n";
    }
291358
def pretty_print(self):
        
        return u"%s\t%.3f\t%.3f\t%s" % (
            (self.identifier or u""),
            (self.begin if self.begin is not None else TimeValue("-2.000")),
            (self.end if self.end is not None else TimeValue("-1.000")),
            (self.text or u"")
        )
239961
def debug_print_tree( self, spacing='' ):
        '''  '''
        print (spacing+" "+str(self.word_id)+" "+str(self.text))
        if (self.children):
            spacing=spacing+"  "
            for child in self.children:
                child.debug_print_tree(spacing)