Return source code based on tokens.
10150
def untokenize_without_newlines(tokens):
    """"""
    text = ''
    last_row = 0
    last_column = -1

    for t in tokens:
        token_string = t[1]
        (start_row, start_column) = t[2]
        (end_row, end_column) = t[3]

        if start_row > last_row:
            last_column = 0
        if (
            (start_column > last_column or token_string == '\n') and
            not text.endswith(' ')
        ):
            text += ' '

        if token_string != '\n':
            text += token_string

        last_row = end_row
        last_column = end_column

    return text.rstrip()
79861
def gen_source(self, ast, name, customize, is_lambda=False, returnNone=False):
        """"""

        rn = self.return_none
        self.return_none = returnNone
        old_name = self.name
        self.name = name
        # if code would be empty, append 'pass'
        if len(ast) == 0:
            self.println(self.indent, 'pass')
        else:
            self.customize(customize)
            if is_lambda:
                self.write(self.traverse(ast, is_lambda=is_lambda))
            else:
                self.text = self.traverse(ast, is_lambda=is_lambda)
                self.println(self.text)
        self.name = old_name
        self.return_none = rn
597213
def evaluate_tokens(tokens):
    """"""
    if isinstance(tokens, str):
        return tokens
    elif isinstance(tokens, ParseResults):
        # evaluate the list portion of the ParseResults
        toklist, name, asList, modal = tokens.__getnewargs__()
        new_toklist = [evaluate_tokens(toks) for toks in toklist]
        new_tokens = ParseResults(new_toklist, name, asList, modal)
        # evaluate the dictionary portion of the ParseResults
        new_tokdict = {}
        for name, occurrences in tokens._ParseResults__tokdict.items():
            new_occurences = []
            for value, position in occurrences:
                if isinstance(value, ParseResults) and value._ParseResults__toklist == toklist:
                    new_value = new_tokens
                else:
                    try:
                        new_value = new_toklist[toklist.index(value)]
                    except ValueError:
                        complain(lambda: CoconutInternalException("inefficient reevaluation of tokens: {} not in {}".format(
                            value,
                            toklist,
                        )))
                        new_value = evaluate_tokens(value)
                new_occurences.append(_ParseResultsWithOffset(new_value, position))
            new_tokdict[name] = occurrences
        new_tokens._ParseResults__accumNames.update(tokens._ParseResults__accumNames)
        new_tokens._ParseResults__tokdict.update(new_tokdict)
        return new_tokens
    elif isinstance(tokens, ComputationNode):
        return tokens.evaluate()
    elif isinstance(tokens, (list, tuple)):
        return [evaluate_tokens(inner_toks) for inner_toks in tokens]
    else:
        raise CoconutInternalException("invalid computation graph tokens", tokens)
736224
def parse_string(self, text, debug=False):
        """"""
        tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline)
        return self.parse_tokens(tokens, debug)
1207287
function(source) {
    var proto = Lexer.prototype;
    var tokens = source.match(proto.regs.token);
    for (var i = 0, l = tokens.length; i < l; i ++) {
        if (proto.regs.strip.test(tokens[i])) {
            tokens.splice(i, 1);
        }
    }
    for (var n in proto) tokens[n] = proto[n];
    tokens.index = 0;
    return tokens;
}
178103
def get_tokens(source):
    
    if not source.endswith('\n'):
        source += '\n'

    # Cache various valid character sets for speed.
    valid_identifier_first_chars = VALID_IDENTIFIER_FIRST_CHARS
    valid_identifier_chars = VALID_IDENTIFIER_CHARS
    hex_digits = HEX_DIGITS
    int_or_float_digits = INT_OR_FLOAT_DIGITS
    int_or_float_digits2 = int_or_float_digits | set('.')

    # Ignore tokens while in a #if 0 block.
    count_ifs = 0

    i = 0
    end = len(source)
    while i < end:
        # Skip whitespace.
        while i < end and source[i].isspace():
            i += 1
        if i >= end:
            return

        token_type = UNKNOWN
        start = i
        c = source[i]
        # Find a string token.
        if c in valid_identifier_first_chars or c == '_':
            token_type = NAME
            while source[i] in valid_identifier_chars:
                i += 1
            # String and character constants can look like a name if
            # they are something like L"".
            if source[i] == "'" and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_char(source, start, i)
            elif source[i] == '"' and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_string(source, i)
        elif c == '/' and source[i + 1] == '/':  # Find // comments.
            i = _find(source, '\n', i)
            continue
        elif c == '/' and source[i + 1] == '*':  # Find /* comments. */
            i = _find(source, '*/', i) + 2
            continue
        elif c in '<>':                          # Handle '<' and '>' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            # Do not merge '>>' or '>>=' into a single token
            if new_ch == c and c != '>':
                i += 1
                new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in ':+-&|=':                      # Handle 'XX' and 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == c:
                i += 1
            elif c == '-' and new_ch == '>':
                i += 1
            elif new_ch == '=':
                i += 1
        elif c in '!*^%/':                       # Handle 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in '()[]{}~?;.,':                 # Handle single char tokens.
            token_type = SYNTAX
            i += 1
            if c == '.' and source[i].isdigit():
                token_type = CONSTANT
                i += 1
                while source[i] in int_or_float_digits:
                    i += 1
                # Handle float suffixes.
                for suffix in ('l', 'f'):
                    if suffix == source[i:i + 1].lower():
                        i += 1
                        break
        elif c.isdigit():                        # Find integer.
            token_type = CONSTANT
            if c == '0' and source[i + 1] in 'xX':
                # Handle hex digits.
                i += 2
                while source[i] in hex_digits:
                    i += 1
            else:
                while source[i] in int_or_float_digits2:
                    i += 1
            # Handle integer (and float) suffixes.
            if source[i].isalpha():
                for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'):
                    size = len(suffix)
                    if suffix == source[i:i + size].lower():
                        i += size
                        break
        elif c == '"':                           # Find string.
            token_type = CONSTANT
            i = _get_string(source, i)
        elif c == "'":                           # Find char.
            token_type = CONSTANT
            i = _get_char(source, start, i)
        elif c == '#':                           # Find pre-processor command.
            token_type = PREPROCESSOR
            got_if = source[i:i + 3] == '#if'
            if count_ifs and source[i:i + 6] == '#endif':
                count_ifs -= 1
                if count_ifs == 0:
                    source = source[:i].ljust(i + 6) + source[i + 6:]
                    continue

            # Handle preprocessor statements (\ continuations).
            while True:
                i1 = source.find('\n', i)
                i2 = source.find('//', i)
                i3 = source.find('/*', i)
                i4 = source.find('"', i)
                # Get the first important symbol (newline, comment, EOF/end).
                i = min([x for x in (i1, i2, i3, i4, end) if x != -1])

                # Handle comments in #define macros.
                if i == i3:
                    i = _find(source, '*/', i) + 2
                    source = source[:i3].ljust(i) + source[i:]
                    continue

                # Handle #include "dir//foo.h" properly.
                if source[i] == '"':
                    i = _find(source, '"', i + 1) + 1
                    continue

                # Keep going if end of the line and the line ends with \.
                if i == i1 and source[i - 1] == '\\':
                    i += 1
                    continue

                if got_if:
                    begin = source.find('(', start, i)
                    if begin == -1:
                        begin = source.find(' ', start)
                    begin = begin + 1
                    s1 = source.find(' ', begin)
                    s2 = source.find(')', begin)
                    s3 = source.find('\n', begin)
                    s = min([x for x in (s1, s2, s3, end) if x != -1])

                    condition = source[begin:s]
                    if (
                        count_ifs or
                        condition == '0' or
                        condition == '__OBJC__'
                    ):
                        count_ifs += 1
                break
        elif c == '\\':                          # Handle \ in code.
            # This is different from the pre-processor \ handling.
            i += 1
            continue
        elif count_ifs:
            # Ignore bogus code when we are inside an #if block.
            i += 1
            continue
        else:
            raise TokenError("unexpected token '{0}'".format(c))

        if count_ifs:
            continue

        assert i > 0
        yield Token(token_type, source[start:i], start, i)
1153650
def get_prompt_tokens(_):
    """"""
    namespace = q(r'\d')
    if namespace == '.':
        namespace = ''
    return [(Token.Generic.Prompt, 'q%s)' % namespace)]
1229925
def get_source_code(filename, lineno, module_globals):
    
    import linecache
    linecache.checkcache(filename)
    source_code = linecache.getline(filename, lineno, module_globals)
    # In case of a multi-line statement, lineno is usually the last line.
    # We are checking for missing open brackets and add earlier code lines.
    while is_source_code_missing_open_brackets(source_code):
        if lineno <= 0:
            break
        lineno -= 1
        source_code = "".join([linecache.getline(filename, lineno, module_globals), source_code])
    return source_code
1345458
def match_tokens(expected_tokens):
    """"""
    if isinstance(expected_tokens, Token):
        # Match a single token.
        def _grammar_func(tokens):
            try:
                next_token = next(iter(tokens))
            except StopIteration:
                return

            if next_token == expected_tokens:
                return TokenMatch(None, next_token.value, (next_token,))

    elif isinstance(expected_tokens, tuple):
        # Match multiple tokens.
        match_len = len(expected_tokens)
        def _grammar_func(tokens):
            upcoming = tuple(itertools.islice(tokens, match_len))
            if upcoming == expected_tokens:
                return TokenMatch(None, None, upcoming)
    else:
        raise TypeError(
            "'expected_tokens' must be an instance of Token or a tuple "
            "thereof. Got %r." % expected_tokens)

    return _grammar_func
282741
def keyword(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == "symbol" and token.value.lower() == expected:
        return TokenMatch(None, token.value, (token,))
479711
def code(self):
    """"""
    return compile(self.source(), self.full_path, 'exec', flags=0, dont_inherit=True)
1051314
def iter_source_code(paths, config, skipped):
    """"""
    for path in paths:
        if os.path.isdir(path):
            if should_skip(path, config, os.getcwd()):
                skipped.append(path)
                continue

            for dirpath, dirnames, filenames in os.walk(path, topdown=True):
                for dirname in list(dirnames):
                    if should_skip(dirname, config, dirpath):
                        skipped.append(dirname)
                        dirnames.remove(dirname)
                for filename in filenames:
                    if filename.endswith('.py'):
                        if should_skip(filename, config, dirpath):
                            skipped.append(filename)
                        else:
                            yield os.path.join(dirpath, filename)
        else:
            yield path
1164613
def get_source_code(items):
    
    sources = []
    prev_class_name = None

    for func in items:
        try:
            lines, lineno = inspect.getsourcelines(func)
        except TypeError:
            continue

        if not lines:
            continue

        src = "\n".join(line.rstrip() for line in lines)
        src = textwrap.dedent(src)

        class_name = None
        if inspect.ismethod(func):
            # Add class name
            if hasattr(func, 'im_class'):
                class_name = func.im_class.__name__
            elif hasattr(func, '__qualname__'):
                names = func.__qualname__.split('.')
                if len(names) > 1:
                    class_name = names[-2]

        if class_name and prev_class_name != class_name:
            src = "class {0}:\n    {1}".format(
                class_name, src.replace("\n", "\n    "))
        elif class_name:
            src = "    {1}".format(
                class_name, src.replace("\n", "\n    "))

        sources.append(src)
        prev_class_name = class_name

    return "\n\n".join(sources).rstrip()
500911
def check_py(self, version, name, original, loc, tokens):
        """"""
        internal_assert(len(tokens) == 1, "invalid " + name + " tokens", tokens)
        if self.target_info < get_target_info(version):
            raise self.make_err(CoconutTargetError, "found Python " + ".".join(version) + " " + name, original, loc, target=version)
        else:
            return tokens[0]
117393
def main():
    """"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        'githubtoken',
        help="Github OAuth token, see https://developer.github.com/v3/oauth/")
    parser.add_argument('destination', help="location of the downloaded repos")
    parser.add_argument(
        '-n', '--nbrepo', help="number of repositories per language",
        type=int, default=1000)
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    destination = Path(args.destination)
    nb_repos = args.nbrepo
    token = args.githubtoken

    languages = config_dict('languages.json')
    destination.mkdir(exist_ok=True)

    for pos, language in enumerate(sorted(languages), 1):
        LOGGER.info("Step %.2f%%, %s", 100 * pos / len(languages), language)
        LOGGER.info("Fetch %d repos infos for language %s", nb_repos, language)
        repos = _retrieve_repo_details(language, nb_repos, token)
        LOGGER.info("%d repos details kept. Downloading", len(repos))
        _download_repos(language, repos, destination)
        LOGGER.info("Language %s repos downloaded", language)

    LOGGER.debug("Exit OK")
1158412
function(code) {

		var tokens = new TokenStream(code),
			hasCRLF = code.indexOf("\r") > -1,
			lines = code.split(/\r?\n/),
			line,
			token,
			tt,
			replacement,
			colAdjust = 0,
			lastLine = 0;

		while ((tt = tokens.get()) !== 0) {
			token = tokens.token();

			if (tt === Tokens.URI) { // URI

				if (lastLine !== token.startLine) {
					colAdjust = 0;
					lastLine = token.startLine;
				}

				replacement = this.replacer(token.value.replace(URL_PARENS, ""));

				// 5 is for url() characters
				line = lines[token.startLine - 1];
				lines[token.startLine - 1] = line.substring(0, token.startCol + colAdjust - 1) +
					"url(" + replacement + ")" + line.substring(token.endCol + colAdjust - 1);
				colAdjust += ((replacement.length + 5) - token.value.length);
			}
		}

		return lines.join(hasCRLF ? "\r\n" : "\n");
	}
808015
def sourcelines(self):
        """"""
        # get sourcecode from loader or file
        source = None
        if self.loader is not None:
            try:
                if hasattr(self.loader, 'get_source'):
                    source = self.loader.get_source(self.module)
                elif hasattr(self.loader, 'get_source_by_code'):
                    source = self.loader.get_source_by_code(self.code)
            except Exception:
                # we munch the exception so that we don't cause troubles
                # if the loader is broken.
                pass

        if source is None:
            try:
                f = open(self.filename)
            except IOError:
                return []
            try:
                source = f.read()
            finally:
                f.close()

        # already unicode?  return right away
        if isinstance(source, str):
            return source.splitlines()

        # yes. it should be ascii, but we don't want to reject too many
        # characters in the debugger if something breaks
        charset = 'utf-8'
        if source.startswith(UTF8_COOKIE):
            source = source[3:]
        else:
            for idx, match in enumerate(_line_re.finditer(source)):
                match = _line_re.search(match.group())
                if match is not None:
                    charset = match.group(1)
                    break
                if idx > 1:
                    break

        # on broken cookies we fall back to utf-8 too
        try:
            codecs.lookup(charset)
        except LookupError:
            charset = 'utf-8'

        return source.decode(charset, 'replace').splitlines()