Scan actions for each commited file in patch set
14219
def scan_patch(project, patch_file, binaries, ips, urls, file_audit_list,
               file_audit_project_list, flag_list, ignore_list, file_ignore,
               ignore_directories, url_ignore, ip_ignore, apikey):

    
    global failure
    split_path = patch_file.split(project + '/', 1)[-1]

    if not any(x in split_path for x in ignore_directories):
        if is_binary(patch_file) and binaries:
            hashlist = get_lists.GetLists()
            binary_hash = hashlist.binary_hash(project, split_path)

            with open(patch_file, 'rb') as afile:
                hasher = hashlib.sha256()
                buf = afile.read()
                hasher.update(buf)
                sha256hash = hasher.hexdigest()

            if sha256hash in binary_hash:

                logger.info('Found matching file hash for: %s',
                            patch_file)
            else:
                logger.info('sha256hash: %s', sha256hash)
                logger.error('Non Whitelisted Binary file: %s',
                             patch_file)

                scan_binary(patch_file, project, sha256hash, apikey)

                failure = True

            with open(reports_dir + "binaries-" + project + ".log", "a") \
                    as gate_report:
                gate_report.write('Non Whitelisted Binary file: {0}\n'.
                                  format(patch_file))
        else:
            # Check file names / extensions
            if file_audit_list.search(patch_file) and not \
                    file_audit_project_list.search(patch_file):
                match = file_audit_list.search(patch_file)
                logger.error('Blacklisted file: %s', patch_file)
                logger.error('Matched String: %s', match.group())
                failure = True
                with open(reports_dir + "file-names_" + project + ".log", "a") \
                        as gate_report:
                    gate_report.write('Blacklisted file: {0}\n'.
                                      format(patch_file))
                    gate_report.write('Matched String: {0}'.
                                      format(match.group()))

            # Open file to check for blacklisted content
            if not is_binary(patch_file):
                try:
                    fo = open(patch_file, 'r')
                    lines = fo.readlines()
                    file_exists = True
                except IOError:
                    file_exists = False

                if file_exists and not patch_file.endswith(tuple(file_ignore)):
                    for line in lines:
                        # Find IP Addresses and send for report to Virus Total
                        if ips:
                            ipaddr = re.findall(r'(?:\d{1,3}\.)+(?:\d{1,3})', line)
                            if ipaddr:
                                ipaddr = ipaddr[0]
                                if re.search(ip_ignore, ipaddr):
                                        logger.info('%s is in IP ignore list.', ipaddr)
                                else:
                                    try:
                                        ipaddress.ip_address(ipaddr).is_global
                                        scan_ipaddr(ipaddr, apikey)
                                    except:
                                        pass  # Ok to pass here, as this captures the odd string which is not an IP Address

                        #  Check for URLs and send for report to Virus Total
                        if urls:
                            url = re.search("(?P<url>https?://[^\s]+)", line) or re.search("(?P<url>www[^\s]+)", line)
                            if url:
                                url = url.group("url")
                                if re.search(url_ignore, url):
                                    logger.info('%s is in URL ignore list.', url)
                                else:
                                    scan_url(url, apikey)

                        #  Perform search within text files
                        for key, value in flag_list.items():
                            regex = value['regex']
                            desc = value['desc']
                            if re.search(regex, line) and not re.search(
                                    ignore_list, line):
                                logger.error('File contains violation: %s', patch_file)
                                logger.error('Flagged Content: %s', line.rstrip())
                                logger.error('Rationale: %s', desc.rstrip())
                                failure = True
                                with open(reports_dir + "contents_" + project + ".log", "a") as gate_report:
                                    gate_report.write('File contains violation: {0}\n'.format(patch_file))
                                    gate_report.write('Flagged Content: {0}'.format(line))
                                    gate_report.write('Matched Regular Exp: {0}\n'.format(regex))
                                    gate_report.write('Rationale: {0}\n'.format(desc.rstrip()))
984064
def current_files
      if commit_range.is_range?
        execute("git #{git_directory_params} -c diff.renames=0 -c diff.renameLimit=1000 diff -M -C -c --name-only --ignore-submodules=all --diff-filter=AM #{encoding_opt} #{default_params} #{commit_range.to_s}") do |result|
          filter_files(result.to_s.split(/\n/))
        end
      else
        submodules = current_submodules
        execute("git #{git_directory_params} ls-tree -r #{commit_range.to_s} --name-only") do |result|
          filter_files(result.to_s.split(/\n/).select { |f| !submodules.index(f) })
        end
      end
    end
298035
def files(obj, commit='HEAD', skip_merge_commits=False):
    """"""
    from ..kwalitee import check_file, SUPPORTED_FILES
    from ..hooks import run
    options = obj.options
    repository = obj.repository

    if options.get('colors') is not False:
        colorama.init(autoreset=True)
        reset = colorama.Style.RESET_ALL
        yellow = colorama.Fore.YELLOW
        green = colorama.Fore.GREEN
        red = colorama.Fore.RED
    else:
        reset = yellow = green = red = ''

    try:
        sha = 'oid'
        commits = _pygit2_commits(commit, repository)
    except ImportError:
        try:
            sha = 'hexsha'
            commits = _git_commits(commit, repository)
        except ImportError:
            click.echo(
                'To use this feature, please install pygit2. GitPython will '
                'also work but is not recommended (python <= 2.7 only).',
                file=sys.stderr)
            click.exit(2)

    template = '{0}commit {{commit.{1}}}{2}\n\n'.format(yellow, sha, reset)
    template += '{message}{errors}\n'

    error_template = '\n{0}{{filename}}\n{1}{{errors}}{0}'.format(reset, red)
    no_errors = ['\n{0}Everything is OK.{1}'.format(green, reset)]
    msg_file_excluded = '\n{0}{{filename}} excluded.{1}'.format(yellow, reset)

    def _get_files_modified(commit):
        """Get the list of modified files that are Python or Jinja2."""
        cmd = "git show --no-commit-id --name-only --diff-filter=ACMRTUXB {0}"
        _, files_modified, _ = run(cmd.format(commit))

        extensions = [re.escape(ext)
                      for ext in list(SUPPORTED_FILES) + [".rst"]]
        test = "(?:{0})$".format("|".join(extensions))
        return list(filter(lambda f: re.search(test, f), files_modified))

    def _ensure_directory(filename):
        dir_ = os.path.dirname(filename)
        if not os.path.exists(dir_):
            os.makedirs(dir_)

    def _format_errors(args):
        filename, errors = args
        if errors is None:
            return msg_file_excluded.format(filename=filename)
        else:
            return error_template.format(filename=filename, errors='\n'.join(
                errors if len(errors) else no_errors))

    count = 0
    ident = '    '
    re_line = re.compile('^', re.MULTILINE)
    for commit in commits:
        if skip_merge_commits and _is_merge_commit(commit):
            continue
        message = commit.message
        commit_sha = getattr(commit, sha)
        tmpdir = mkdtemp()
        errors = {}
        try:
            for filename in _get_files_modified(commit):
                cmd = "git show {commit_sha}:{filename}"
                _, out, _ = run(cmd.format(commit_sha=commit_sha,
                                           filename=filename),
                                raw_output=True)

                destination = os.path.join(tmpdir, filename)
                _ensure_directory(destination)

                with open(destination, 'w+') as f:
                    f.write(out)

                errors[filename] = check_file(destination, **options)
        finally:
            shutil.rmtree(tmpdir, ignore_errors=True)

        message = re.sub(re_line, ident, message)
        if len(errors):
            count += 1
            errors = map(_format_errors, errors.items())
        else:
            errors = no_errors

        click.echo(template.format(commit=commit,
                                   message=message.encode('utf-8'),
                                   errors='\n'.join(errors)))

    if min(count, 1):
        raise click.Abort
640092
def scan(repos, options):
    """"""
    ignore_set = set()
    repos = repos[::-1]  # Create a queue we can push and pop from
    while repos:
        directory, dotdir = repos.pop()
        ignore_this = any(pat in directory for pat in options.ignore_patterns)
        if ignore_this:
            if options.verbose:
                output(b'Ignoring repo: %s' % directory)
                output(b'')
            continue

        vcsname, get_status = SYSTEMS[dotdir]
        lines, subrepos = get_status(directory, ignore_set, options)

        # We want to tackle subrepos immediately after their repository,
        # so we put them at the front of the queue.
        subrepos = [(os.path.join(directory, r), dotdir) for r in subrepos]
        repos.extend(reversed(subrepos))

        if lines is None:  # signal that we should ignore this one
            continue
        if lines or options.verbose:
            output(b'%s - %s' % (directory, vcsname))
            for line in lines:
                output(line)
            output(b'')
115198
def main(commit_only=True):
    
    global TEMP_FOLDER
    exit_code = 0
    hook_checks = HookConfig(get_config_file())
    with files_to_check(commit_only) as files:
        for name, mod in checks():
            default = getattr(mod, 'DEFAULT', 'off')
            if hook_checks.is_enabled(name, default=default):
                if hasattr(mod, 'REQUIRED_FILES'):
                    for filename in mod.REQUIRED_FILES:
                        if os.path.isfile(filename):
                            try:
                                shutil.copy(filename, TEMP_FOLDER)
                            except shutil.Error:
                                # Copied over by a previous check
                                continue
                args = hook_checks.arguments(name)

                tmp_files = [os.path.join(TEMP_FOLDER, f) for f in files]

                if args:
                    errors = mod.run(tmp_files, TEMP_FOLDER, args)
                else:
                    errors = mod.run(tmp_files, TEMP_FOLDER)
                if errors:
                    title_print("Checking {0}".format(name))
                    print((errors.replace(TEMP_FOLDER + "/", '')))
                    print("")
                    exit_code = 1

    if exit_code == 1:
        title_print("Rejecting commit")
    return exit_code
808481
def batch_commit(self, *, do_deletes=False):
        '''
        
        '''
        try:
            yield
        except Exception as exc:
            raise exc
        else:
            for key, value in self.cache.items():
                if value is not DELETED:
                    self.wrapped_db[key] = value
                elif do_deletes:
                    self.wrapped_db.pop(key, None)
                # if do_deletes is False, ignore deletes to underlying db
        finally:
            self.cache = {}
171005
function analyzePatch(patch) {
  const patchLines = patch.split('\n').reduce((reducedPatch, currentLine) => {
    if (currentLine.match(/^[-+]/)) {
      return reducedPatch.concat(currentLine.replace(/^[-+]+\s*/, ''));
    }
    return reducedPatch;
  }, []);
  return {
    lines: patchLines.length,
    chars: patchLines.join('').length
  };
}
448454
def __update_action(self, revision):
        

        patch = revision.get("patch")
        if patch.get("_id"):
            del patch["_id"]

        update_response = yield self.collection.patch(revision.get("master_id"), self.__make_storeable_patch_patchable(patch))

        if update_response.get("n") == 0:
            raise RevisionNotFoundException()
1045276
def commit(*args):
    """"""
    parser = argparse.ArgumentParser(prog="%s %s" % (__package__, commit.__name__), description=commit.__doc__)
    parser.add_argument('FILENAME', help="file(s) to commit", nargs="*", default=['.'])
    args = parser.parse_args(args)

    config = FragmentsConfig()

    for s, curr_path in _iterate_over_files(args.FILENAME, config, statuses='MAD'):
        key = os.path.relpath(curr_path, config.root)
        if key not in config['files']:
            yield "Could not commit '%s' because it is not being followed" % os.path.relpath(curr_path)
            continue

        if s in 'MA':
            repo_path = os.path.join(config.directory, config['files'][key])
            with _smart_open(repo_path, 'w') as repo_file:
                with _smart_open(curr_path, 'r') as curr_file:
                    repo_file.write(curr_file.read())
            os.utime(repo_path, os.stat(curr_path)[7:9])
            yield "'%s' committed" % os.path.relpath(curr_path)
        elif s == 'D':
            yield "Could not commit '%s' because it has been removed, instead revert or forget it" % os.path.relpath(curr_path)
        elif s == ' ':
            yield "Could not commit '%s' because it has not been changed" % os.path.relpath(curr_path)
257257
def git_check():
    """"""
    git_status = subprocess.check_output(['git', 'status', '--porcelain'])
    if len(git_status) is 0:
        print(Fore.GREEN + 'All changes committed' + Style.RESET_ALL)
    else:
        exit(Fore.RED + 'Please commit all files to continue')
576989
def changed_lines(self):
        
        lines = []
        file_name = ''
        line_number = 0
        patch_position = -1
        found_first_information_line = False

        for i, content in enumerate(self.body.splitlines()):
            range_information_match = RANGE_INFORMATION_LINE.search(content)
            file_name_line_match = FILE_NAME_LINE.search(content)

            if file_name_line_match:
                file_name = file_name_line_match.group('file_name')
                found_first_information_line = False
            elif range_information_match:
                line_number = int(range_information_match.group('line_number'))
                if not found_first_information_line:
                    # This is the first information line. Set patch position to 1 and start counting
                    patch_position = 0
                    found_first_information_line = True
            elif MODIFIED_LINE.search(content):
                line = {
                    'file_name': file_name,
                    'content': content,
                    'line_number': line_number,
                    'position': patch_position
                }

                lines.append(line)
                line_number += 1
            elif NOT_REMOVED_OR_NEWLINE_WARNING.search(content) or content == '':
                line_number += 1

            patch_position += 1

        return lines
1155500
def pre_commit_hook(argv):
    """"""
    options = get_options()
    # Check if the repo has a configuration repo
    options.update(_read_local_kwalitee_configuration())

    files = []
    for filename in _get_files_modified():
        # get the staged version of the file and
        # write the staged version to temp dir with its full path to
        # avoid overwriting files with the same name
        _, stdout, _ = run("git show :{0}".format(filename), raw_output=True)
        files.append((filename, stdout))

    errors = _pre_commit(files, options)

    for error in errors:
        if hasattr(error, "decode"):
            error = error.decode()
        click.echo(error, file=sys.stderr)

    if errors:
        click.echo(
            "Aborting commit due to kwalitee errors (override with "
            "'git commit --no-verify').",
            file=sys.stderr)
        raise click.Abort
    return 0
1162764
def git_hook(error=True):
    """"""
    _, files_modified, _ = run("git diff-index --cached --name-only HEAD")

    options = parse_options()
    setup_logger(options)
    if sys.version_info >= (3,):
        candidates = [f.decode('utf-8') for f in files_modified]
    else:
        candidates = [str(f) for f in files_modified]

    if candidates:
        process_paths(options, candidates=candidates, error=error)
615122
def _patch_file(path, content):
    """"""
    f = open(path)
    existing_content = f.read()
    f.close()
    if existing_content == content:
        # already patched
        log.warn('Already patched.')
        return False
    log.warn('Patching...')
    _rename_path(path)
    f = open(path, 'w')
    try:
        f.write(content)
    finally:
        f.close()
    return True
561560
def refresh( self ):
        
        self.clear()
        
        for i, filename in enumerate(self.filenames()):
            name   = '%i. %s' % (i+1, os.path.basename(filename))
            action = self.addAction(name)
            action.setData(wrapVariant(filename))
969893
def all_files(self):
        """"""
        return set([entry.decode() for entry, _ in self.git.open_index().items()])
526428
def _update_tokens(self, write_token=True, override=None,
                       patch_skip=False):
        """"""
        next_token = next(self.tokens)

        patch_value = ''
        patch_tokens = ''

        if self.pfile and write_token:
            token = override if override else self.token
            patch_value += token

        while next_token[0] in self.comment_tokens + whitespace:
            if self.pfile:
                if next_token[0] in self.comment_tokens:
                    while not next_token == '\n':
                        patch_tokens += next_token
                        next_token = next(self.tokens)
                patch_tokens += next_token

            # Several sections rely on StopIteration to terminate token search
            # If that occurs, dump the patched tokens immediately
            try:
                next_token = next(self.tokens)
            except StopIteration:
                if not patch_skip or next_token in ('=', '(', '%'):
                    patch_tokens = patch_value + patch_tokens

                if self.pfile:
                    self.pfile.write(patch_tokens)
                raise

        # Write patched values and whitespace + comments to file
        if not patch_skip or next_token in ('=', '(', '%'):
            patch_tokens = patch_value + patch_tokens

        if self.pfile:
            self.pfile.write(patch_tokens)

        # Update tokens, ignoring padding
        self.token, self.prior_token = next_token, self.token