Remove comments and docstrings from Python `source`, preserving line
    numbers and syntax of empty blocks.

    :param str source:
        The source to minimize.

    :returns str:
        The minimized source.
765234
def minimize_source(source):
    
    source = mitogen.core.to_text(source)
    tokens = tokenize.generate_tokens(StringIO(source).readline)
    tokens = strip_comments(tokens)
    tokens = strip_docstrings(tokens)
    tokens = reindent(tokens)
    return tokenize.untokenize(tokens)
1284000
def remove_comments(self, recursive=True):
        
        self.text = texutils.remove_comments(self.text)
        if recursive:
            for path, document in self._children.iteritems():
                document.remove_comments(recursive=True)
632300
def remove_excluded(self):
        
        # import yaml
        # print yaml.dump({k:v.__json__() for k,v in self.sources.items()}, default_flow_style=False)
        sources = list(self.sources.values())
        for src in sources:
            if src.excluded:
                del self.sources[src.name]
            src.imports = [m for m in src.imports if not self._exclude(m)]
            src.imported_by = [m for m in src.imported_by if not self._exclude(m)]
1289601
def _strip_top_comments(lines):
        """"""
        lines = copy.copy(lines)
        while lines and lines[0].startswith("#"):
            lines = lines[1:]
        return "\n".join(lines)
993288
def clean(docs=False, bytecode=False, extra=''):
    ''''''
    patterns = ['build', 'dist', 'cover', 'docs/_build', '**/*.pyc', '*.egg-info', '.tox', '**/__pycache__']
    for pattern in patterns:
        print('Removing {0}'.format(pattern))
        lrun('rm -rf {0}'.format(pattern))
1093192
def fix_e265(source, aggressive=False):  # pylint: disable=unused-argument
    """"""
    if '#' not in source:
        # Optimization.
        return source

    ignored_line_numbers = multiline_string_lines(
        source,
        include_docstrings=True) | set(commented_out_code_lines(source))

    fixed_lines = []
    sio = io.StringIO(source)
    for (line_number, line) in enumerate(sio.readlines(), start=1):
        if (
            line.lstrip().startswith('#') and
            line_number not in ignored_line_numbers and
            not pycodestyle.noqa(line)
        ):
            indentation = _get_indentation(line)
            line = line.lstrip()

            # Normalize beginning if not a shebang.
            if len(line) > 1:
                pos = next((index for index, c in enumerate(line)
                            if c != '#'))
                if (
                    # Leave multiple spaces like '#    ' alone.
                    (line[:pos].count('#') > 1 or line[1].isalnum()) and
                    # Leave stylistic outlined blocks alone.
                    not line.rstrip().endswith('#')
                ):
                    line = '# ' + line.lstrip('# \t')

            fixed_lines.append(indentation + line)
        else:
            fixed_lines.append(line)

    return ''.join(fixed_lines)
1328020
def __clean_and_tokenize(self, doc_list):
        
        # Some repositories fill entire documentation in description. We ignore
        # such repositories for cleaner tokens.
        doc_list = filter(
            lambda x: x is not None and len(x) <= GitSuggest.MAX_DESC_LEN,
            doc_list,
        )

        cleaned_doc_list = list()

        # Regular expression to remove out all punctuations, numbers and other
        # un-necessary text substrings like emojis etc.
        tokenizer = RegexpTokenizer(r"[a-zA-Z]+")

        # Get stop words.
        stopwords = self.__get_words_to_ignore()

        # Get english words.
        dict_words = self.__get_words_to_consider()

        for doc in doc_list:
            # Lowercase doc.
            lower = doc.lower()

            # Tokenize removing numbers and punctuation.
            tokens = tokenizer.tokenize(lower)

            # Include meaningful words.
            tokens = [tok for tok in tokens if tok in dict_words]

            # Remove stopwords.
            tokens = [tok for tok in tokens if tok not in stopwords]

            # Filter Nones if any are introduced.
            tokens = [tok for tok in tokens if tok is not None]

            cleaned_doc_list.append(tokens)

        return cleaned_doc_list
1191313
def multiline_string_lines(source, include_docstrings=False):
    
    line_numbers = set()
    previous_token_type = ''
    try:
        for t in generate_tokens(source):
            token_type = t[0]
            start_row = t[2][0]
            end_row = t[3][0]

            if token_type == tokenize.STRING and start_row != end_row:
                if (
                    include_docstrings or
                    previous_token_type != tokenize.INDENT
                ):
                    # We increment by one since we want the contents of the
                    # string.
                    line_numbers |= set(range(1 + start_row, 1 + end_row))

            previous_token_type = token_type
    except (SyntaxError, tokenize.TokenError):
        pass

    return line_numbers
73077
def delete_edge(self, source: str, target: str):
        
        if self.has_edge(source, target):
            self.remove_edge(source, target)
153609
def _strip_top_comments(lines: Sequence[str], line_separator: str) -> str:
        """"""
        lines = copy.copy(lines)
        while lines and lines[0].startswith("#"):
            lines = lines[1:]
        return line_separator.join(lines)
797014
def parse(source, remove_comments=True, **kw):
    """"""
    return ElementTree.parse(source, SourceLineParser(), **kw)
2192
function minify(src) {
       // remove comments
       src = src.replace(/\/\*[\s\S]*?\*\/|([^\\:]|^)\/\/.*$/gm, "$1"); // Remove leading and trailing whitespace from lines

       src = src.replace(/(\\n\s+)|(\s+\\n)/g, ""); // Remove line breaks

       src = src.replace(/(\\r|\\n)+/g, ""); // Remove unnecessary whitespace

       src = src.replace(/\s*([;,[\](){}\\\/\-+*|^&!=<>?~%])\s*/g, "$1");
       return src;
     }
102051
def remove_source(self, source):
        
        geocode_service = self._get_service_by_name(source[0])
        self._sources.remove(geocode_service(**source[1]))
1113541
def source(self, getline=linecache.getline):
        
        try:
            return getline(self.filename, self.lineno)
        except Exception as exc:
            return "??? NO SOURCE: {!r}".format(exc)
7827
def fix_lines(source_lines, options, filename=''):
    """"""
    # Transform everything to line feed. Then change them back to original
    # before returning fixed source code.
    original_newline = find_newline(source_lines)
    tmp_source = ''.join(normalize_line_endings(source_lines, '\n'))

    # Keep a history to break out of cycles.
    previous_hashes = set()

    if options.line_range:
        # Disable "apply_local_fixes()" for now due to issue #175.
        fixed_source = tmp_source
    else:
        pep8_options = {
            'ignore': options.ignore,
            'select': options.select,
            'max_line_length': options.max_line_length,
            'hang_closing': options.hang_closing,
        }
        sio = io.StringIO(tmp_source)
        contents = sio.readlines()
        results = _execute_pep8(pep8_options, contents)
        codes = {result['id'] for result in results
                 if result['id'] in SELECTED_GLOBAL_FIXED_METHOD_CODES}
        # Apply global fixes only once (for efficiency).
        fixed_source = apply_global_fixes(tmp_source,
                                          options,
                                          filename=filename,
                                          codes=codes)

    passes = 0
    long_line_ignore_cache = set()
    while hash(fixed_source) not in previous_hashes:
        if options.pep8_passes >= 0 and passes > options.pep8_passes:
            break
        passes += 1

        previous_hashes.add(hash(fixed_source))

        tmp_source = copy.copy(fixed_source)

        fix = FixPEP8(
            filename,
            options,
            contents=tmp_source,
            long_line_ignore_cache=long_line_ignore_cache)

        fixed_source = fix.fix()

    sio = io.StringIO(fixed_source)
    return ''.join(normalize_line_endings(sio.readlines(), original_newline))
460906
def strip_magics(source):
    
    filtered=[]
    for line in source.splitlines():
        if not line.startswith('%') or line.startswith('%%'):
            filtered.append(line)
    return '\n'.join(filtered)
627150
def source_to_code(self, nodes, path, *, _optimize=-1):
        """"""
        if not isinstance(nodes, ast.Module):
            nodes = ast.parse(nodes, self.path)
        if self._markdown_docstring:
            nodes = update_docstring(nodes)
        return super().source_to_code(
            ast.fix_missing_locations(self.visit(nodes)), path, _optimize=_optimize
        )