{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to Git via GitPython\n\nFor further information on GitPython see http://gitpython.readthedocs.org/\n\n\"\"\"\n\nimport re\nimport time\nimport os\nimport os.path as op\nimport warnings\nfrom locale import getpreferredencoding\n\n\nfrom itertools import chain\nimport logging\nfrom collections import (\n    OrderedDict,\n    namedtuple\n)\nfrom os import linesep\nfrom os.path import (\n    join as opj,\n    exists,\n    normpath,\n    isabs,\n    commonprefix,\n    relpath,\n    realpath,\n    dirname,\n    basename,\n    curdir,\n    pardir,\n    sep\n)\n\nimport posixpath\nfrom functools import wraps\nfrom weakref import WeakValueDictionary\n\nimport git as gitpy\nfrom gitdb.exc import BadName\nfrom git.exc import (\n    GitCommandError,\n    NoSuchPathError,\n    InvalidGitRepositoryError\n)\nfrom datalad.log import log_progress\nfrom datalad.support.due import due, Doi\n\nfrom datalad import ssh_manager\nfrom datalad.cmd import (\n    WitlessRunner,\n    WitlessProtocol,\n    GitRunner,\n    BatchedCommand,\n    run_gitcommand_on_file_list_chunks,\n    StdOutErrCapture,\n)\nfrom datalad.config import (\n    ConfigManager,\n    _parse_gitconfig_dump\n)\n\nfrom datalad.consts import (\n    ADJUSTED_BRANCH_EXPR,\n)\nfrom datalad.dochelpers import exc_str\nimport datalad.utils as ut\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    ensure_list,\n    optional_args,\n    on_windows,\n    getpwd,\n    posix_relpath,\n    ensure_dir,\n    generate_file_chunks,\n    ensure_unicode,\n    split_cmdline,\n)\n\n# imports from same module:\nfrom .external_versions import external_versions\nfrom .exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    GitIgnoreError,\n    InvalidGitReferenceError,\n    MissingBranchError,\n    OutdatedExternalDependencyWarning,\n    PathKnownToRepositoryError,\n)\nfrom .network import (\n    RI,\n    PathRI,\n    is_ssh\n)\nfrom .path import get_parent_paths\nfrom .repo import (\n    PathBasedFlyweight,\n    RepoInterface\n)\n\n#from git.objects.blob import Blob\n\n# shortcuts\n_curdirsep = curdir + sep\n_pardirsep = pardir + sep\n\n\nlgr = logging.getLogger('datalad.gitrepo')\n_lgr_level = lgr.getEffectiveLevel()\nif _lgr_level <= 2:\n    from ..log import LoggerHelper\n    # Let's also enable gitpy etc debugging\n    gitpy_lgr = LoggerHelper(logtarget=\"git\").get_initialized_logger()\n    gitpy_lgr.setLevel(_lgr_level)\n    gitpy_lgr.propagate = True\n\n# Override default GitPython's DB backend to talk directly to git so it doesn't\n# interfere with possible operations performed by gc/repack\ndefault_git_odbt = gitpy.GitCmdObjectDB\n\n# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used.\n# Didn't work as expected on a first try. Probably there is a neatier way to\n# log Exceptions from git commands.\n\n\ndef to_options(**kwargs):\n    \"\"\"Transform keyword arguments into a list of cmdline options\n\n    Parameters\n    ----------\n    split_single_char_options: bool\n\n    kwargs:\n\n    Returns\n    -------\n    list\n    \"\"\"\n    # TODO: borrow_docs!\n\n    return gitpy.Git().transform_kwargs(**kwargs)\n\n\ndef _normalize_path(base_dir, path):\n    \"\"\"Helper to check paths passed to methods of this class.\n\n    Checks whether `path` is beneath `base_dir` and normalizes it.\n    Additionally paths are converted into relative paths with respect to\n    `base_dir`, considering PWD in case of relative paths. This\n    is intended to be used in repository classes, which means that\n    `base_dir` usually will be the repository's base directory.\n\n    Parameters\n    ----------\n    base_dir: str\n        directory to serve as base to normalized, relative paths\n    path: str\n        path to be normalized\n\n    Returns\n    -------\n    str:\n        path, that is a relative path with respect to `base_dir`\n    \"\"\"\n    if not path:\n        return path\n\n    base_dir = realpath(base_dir)  # realpath OK\n    # path = normpath(path)\n    # Note: disabled normpath, because it may break paths containing symlinks;\n    # But we don't want to realpath relative paths, in case cwd isn't the\n    # correct base.\n\n    if isabs(path):\n        # path might already be a symlink pointing to annex etc,\n        # so realpath only its directory, to get \"inline\" with\n        # realpath(base_dir) above\n        path = opj(realpath(dirname(path)), basename(path))  # realpath OK\n    # Executive decision was made to not do this kind of magic!\n    #\n    # elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:\n    #     # If we are inside repository, rebuilt relative paths.\n    #     path = opj(realpath(getpwd()), path)\n    #\n    # BUT with relative curdir/pardir start it would assume relative to curdir\n    #\n    elif path.startswith(_curdirsep) or path.startswith(_pardirsep):\n        path = normpath(opj(realpath(getpwd()), path))  # realpath OK\n    else:\n        # We were called from outside the repo. Therefore relative paths\n        # are interpreted as being relative to self.path already.\n        return path\n\n    if commonprefix([path, base_dir]) != base_dir:\n        raise FileNotInRepositoryError(msg=\"Path outside repository: %s\"\n                                           % path, filename=path)\n\n    return relpath(path, start=base_dir)\n\n\n@optional_args\ndef normalize_path(func):\n    \"\"\"Decorator to provide unified path conversion for a single file\n\n    Unlike normalize_paths, intended to be used for functions dealing with a\n    single filename at a time\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, file_, *args, **kwargs):\n        file_new = _normalize_path(self.path, file_)\n        return func(self, file_new, *args, **kwargs)\n\n    return newfunc\n\n\n@optional_args\ndef normalize_paths(func, match_return_type=True, map_filenames_back=False,\n                    serialize=False):\n    \"\"\"Decorator to provide unified path conversions.\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path or a list of paths at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n\n    Accepts either a list of paths or a single path in a str. Passes a list\n    to decorated function either way, but would return based on the value of\n    match_return_type and possibly input argument.\n\n    If a call to the wrapped function includes normalize_path and it is False\n    no normalization happens for that function call (used for calls to wrapped\n    functions within wrapped functions, while possible CWD is within a\n    repository)\n\n    Parameters\n    ----------\n    match_return_type : bool, optional\n      If True, and a single string was passed in, it would return the first\n      element of the output (after verifying that it is a list of length 1).\n      It makes easier to work with single files input.\n    map_filenames_back : bool, optional\n      If True and returned value is a dictionary, it assumes to carry entries\n      one per file, and then filenames are mapped back to as provided from the\n      normalized (from the root of the repo) paths\n    serialize : bool, optional\n      Loop through files giving only a single one to the function one at a time.\n      This allows to simplify implementation and interface to annex commands\n      which do not take multiple args in the same call (e.g. checkpresentkey)\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, files, *args, **kwargs):\n\n        normalize = _normalize_path if kwargs.pop('normalize_paths', True) \\\n            else lambda rpath, filepath: filepath\n\n        if files:\n            if isinstance(files, str) or not files:\n                files_new = [normalize(self.path, files)]\n                single_file = True\n            elif isinstance(files, list):\n                files_new = [normalize(self.path, path) for path in files]\n                single_file = False\n            else:\n                raise ValueError(\"_files_decorator: Don't know how to handle \"\n                                 \"instance of %s.\" % type(files))\n        else:\n            single_file = None\n            files_new = []\n\n        if map_filenames_back:\n            def remap_filenames(out):\n                \"\"\"Helper to map files back to non-normalized paths\"\"\"\n                if isinstance(out, dict):\n                    assert(len(out) == len(files_new))\n                    files_ = [files] if single_file else files\n                    mapped = out.__class__()\n                    for fin, fout in zip(files_, files_new):\n                        mapped[fin] = out[fout]\n                    return mapped\n                else:\n                    return out\n        else:\n            remap_filenames = lambda x: x\n\n        if serialize:  # and not single_file:\n            result = [\n                func(self, f, *args, **kwargs)\n                for f in files_new\n            ]\n        else:\n            result = func(self, files_new, *args, **kwargs)\n\n        if single_file is None:\n            # no files were provided, nothing we can do really\n            return result\n        elif (result is None) or not match_return_type or not single_file:\n            # If function doesn't return anything or no denormalization\n            # was requested or it was not a single file\n            return remap_filenames(result)\n        elif single_file:\n            if len(result) != 1:\n                # Magic doesn't apply\n                return remap_filenames(result)\n            elif isinstance(result, (list, tuple)):\n                return result[0]\n            elif isinstance(result, dict) and tuple(result)[0] == files_new[0]:\n                # assume that returned dictionary has files as keys.\n                return tuple(result.values())[0]\n            else:\n                # no magic can apply\n                return remap_filenames(result)\n        else:\n            return RuntimeError(\"should have not got here... check logic\")\n\n    return newfunc\n\n\ndef _remove_empty_items(list_):\n    \"\"\"Remove empty entries from list\n\n    This is needed, since some functions of GitPython may convert\n    an empty entry to '.', when used with a list of paths.\n\n    Parameter:\n    ----------\n    list_: list of str\n\n    Returns\n    -------\n    list of str\n    \"\"\"\n    if not isinstance(list_, list):\n        lgr.warning(\n            \"_remove_empty_items() called with non-list type: %s\" % type(list_))\n        return list_\n    return [file_ for file_ in list_ if file_]\n\n\nif \"2.24.0\" <= external_versions[\"cmd:git\"] < \"2.25.0\":\n    # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing\n    # into untracked submodules when multiple pathspecs are given, returning\n    # repositories that are deeper than the first level. This helper filters\n    # these deeper levels out so that save_() doesn't fail trying to add them.\n    #\n    # This regression fixed with upstream's 072a231016 (2019-12-10).\n    def _prune_deeper_repos(repos):\n        firstlevel_repos = []\n        prev = None\n        for repo in sorted(repos):\n            if not (prev and str(repo).startswith(prev)):\n                prev = str(repo)\n                firstlevel_repos.append(repo)\n        return firstlevel_repos\nelse:\n    def _prune_deeper_repos(repos):\n        return repos\n\n\ndef Repo(*args, **kwargs):\n    \"\"\"Factory method around gitpy.Repo to consistently initiate with different\n    backend\n    \"\"\"\n    # TODO: This probably doesn't work as intended (or at least not as\n    #       consistently as intended). gitpy.Repo could be instantiated by\n    #       classmethods Repo.init or Repo.clone_from. In these cases 'odbt'\n    #       would be needed as a parameter to these methods instead of the\n    #       constructor.\n    if 'odbt' not in kwargs:\n        kwargs['odbt'] = default_git_odbt\n    return gitpy.Repo(*args, **kwargs)\n\n\ndef guard_BadName(func):\n    \"\"\"A helper to guard against BadName exception\n\n    Workaround for\n    https://github.com/gitpython-developers/GitPython/issues/768\n    also see https://github.com/datalad/datalad/issues/2550\n    Let's try to precommit (to flush anything flushable) and do\n    it again\n    \"\"\"\n\n    @wraps(func)\n    def wrapped(repo, *args, **kwargs):\n        try:\n            return func(repo, *args, **kwargs)\n        except BadName:\n            repo.precommit()\n            return func(repo, *args, **kwargs)\n\n    return wrapped\n\n\nclass GitProgress(WitlessProtocol):\n    \"\"\"Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    # inform super-class to capture stderr\n    proc_err = True\n\n    _num_op_codes = 10\n    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \\\n        [1 << x for x in range(_num_op_codes)]\n    STAGE_MASK = BEGIN | END\n    OP_MASK = ~STAGE_MASK\n\n    DONE_TOKEN = 'done.'\n    TOKEN_SEPARATOR = ', '\n\n    _known_ops = {\n        COUNTING: (\"Counting\", \"Objects\"),\n        ENUMERATING: (\"Enumerating\", \"Objects\"),\n        COMPRESSING: (\"Compressing\", \"Objects\"),\n        WRITING: (\"Writing\", \"Objects\"),\n        RECEIVING: (\"Receiving\", \"Objects\"),\n        RESOLVING: (\"Resolving\", \"Deltas\"),\n        FINDING_SOURCES: (\"Finding\", \"Sources\"),\n        CHECKING_OUT: (\"Check out\", \"Things\"),\n    }\n\n    __slots__ = ('_seen_ops', '_pbars', '_encoding')\n\n    re_op_absolute = re.compile(r\"(remote: )?([\\w\\s]+):\\s+()(\\d+)()(.*)\")\n    re_op_relative = re.compile(r\"(remote: )?([\\w\\s]+):\\s+(\\d+)% \\((\\d+)/(\\d+)\\)(.*)\")\n\n    def __init__(self, *args):\n        super().__init__(*args)\n        self._encoding = getpreferredencoding(do_setlocale=False)\n        self._unprocessed = None\n\n    def connection_made(self, transport):\n        super().connection_made(transport)\n        self._seen_ops = []\n        self._pbars = set()\n\n    def process_exited(self):\n        # take down any progress bars that were not closed orderly\n        for pbar_id in self._pbars:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished',\n            )\n        super().process_exited()\n\n    def pipe_data_received(self, fd, byts):\n        # progress reports only come from stderr\n        if fd != 2:\n            # let the base class decide what to do with it\n            super().pipe_data_received(fd, byts)\n            return\n        for line in byts.splitlines(keepends=True):\n            # put any unprocessed content back in front\n            line = self._unprocessed + line if self._unprocessed else line\n            self._unprocessed = None\n            if not self._parse_progress_line(line):\n                # anything that doesn't look like a progress report\n                # is retained and returned\n                # in case of partial progress lines, this can lead to\n                # leakage of progress info into the output, but\n                # it is better to enable better (maybe more expensive)\n                # subsequent filtering than hidding lines with\n                # unknown, potentially important info\n                lgr.debug('Non-progress stderr: %s', line)\n                if line.endswith((b'\\r', b'\\n')):\n                    # complete non-progress line, pass on\n                    super().pipe_data_received(fd, line)\n                else:\n                    # an incomplete line, maybe the next batch completes\n                    # it to become a recognizable progress report\n                    self._unprocessed = line\n\n    def _parse_progress_line(self, line):\n        \"\"\"Process a single line\n\n        Parameters\n        ----------\n        line : bytes\n\n        Returns\n        -------\n        bool\n          Flag whether the line was recognized as a Git progress report.\n        \"\"\"\n        # handle\n        # Counting objects: 4, done.\n        # Compressing objects:  50% (1/2)\n        # Compressing objects: 100% (2/2)\n        # Compressing objects: 100% (2/2), done.\n        line = line.decode(self._encoding) if isinstance(line, bytes) else line\n        if line.startswith(('warning:', 'error:', 'fatal:')):\n            return False\n\n        # find escape characters and cut them away - regex will not work with\n        # them as they are non-ascii. As git might expect a tty, it will send them\n        last_valid_index = None\n        for i, c in enumerate(reversed(line)):\n            if ord(c) < 32:\n                # its a slice index\n                last_valid_index = -i - 1\n            # END character was non-ascii\n        # END for each character in line\n        if last_valid_index is not None:\n            line = line[:last_valid_index]\n        # END cut away invalid part\n        line = line.rstrip()\n\n        cur_count, max_count = None, None\n        match = self.re_op_relative.match(line)\n        if match is None:\n            match = self.re_op_absolute.match(line)\n\n        if not match:\n            return False\n        # END could not get match\n\n        op_code = 0\n        _remote, op_name, _percent, cur_count, max_count, message = match.groups()\n\n        # get operation id\n        if op_name == \"Counting objects\":\n            op_code |= self.COUNTING\n        elif op_name == \"Compressing objects\":\n            op_code |= self.COMPRESSING\n        elif op_name == \"Writing objects\":\n            op_code |= self.WRITING\n        elif op_name == 'Receiving objects':\n            op_code |= self.RECEIVING\n        elif op_name == 'Resolving deltas':\n            op_code |= self.RESOLVING\n        elif op_name == 'Finding sources':\n            op_code |= self.FINDING_SOURCES\n        elif op_name == 'Checking out files':\n            op_code |= self.CHECKING_OUT\n        elif op_name == 'Enumerating objects':\n            op_code |= self.ENUMERATING\n        else:\n            # Note: On windows it can happen that partial lines are sent\n            # Hence we get something like \"CompreReceiving objects\", which is\n            # a blend of \"Compressing objects\" and \"Receiving objects\".\n            # This can't really be prevented.\n            lgr.debug(\n                'Output line matched a progress report of an unknown type: %s',\n                line)\n            # TODO investigate if there is any chance that we might swallow\n            # important info -- until them do not flag this line\n            # as progress\n            return False\n        # END handle op code\n\n        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)\n\n        op_props = self._known_ops[op_code]\n\n        # figure out stage\n        if op_code not in self._seen_ops:\n            self._seen_ops.append(op_code)\n            op_code |= self.BEGIN\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n                label=op_props[0],\n                unit=' {}'.format(op_props[1]),\n                total=float(max_count) if max_count else None,\n            )\n            self._pbars.add(pbar_id)\n        # END begin opcode\n\n        if message is None:\n            message = ''\n        # END message handling\n\n        done_progress = False\n        message = message.strip()\n        if message.endswith(self.DONE_TOKEN):\n            op_code |= self.END\n            message = message[:-len(self.DONE_TOKEN)]\n            done_progress = True\n        # END end message handling\n        message = message.strip(self.TOKEN_SEPARATOR)\n\n        if cur_count and max_count:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                message,\n                update=float(cur_count),\n            )\n\n        if done_progress:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n            )\n            self._pbars.discard(pbar_id)\n        return True\n\n\nclass StdOutCaptureWithGitProgress(GitProgress):\n    proc_out = True\n\n\nclass FetchInfo(dict):\n    \"\"\"\n    dict that carries results of a fetch operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n\n    NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \\\n        FAST_FORWARD, ERROR = [1 << x for x in range(8)]\n\n    _re_fetch_result = re.compile(r'^\\s*(.) (\\[?[\\w\\s\\.$@]+\\]?)\\s+(.+) [-> ]+ ([^\\s]+)(    \\(.*\\)?$)?')\n\n    _flag_map = {\n        '!': ERROR,\n        '+': FORCED_UPDATE,\n        '*': 0,\n        '=': HEAD_UPTODATE,\n        ' ': FAST_FORWARD,\n        '-': TAG_UPDATE,\n    }\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        HEAD_UPTODATE: 'uptodate',\n        TAG_UPDATE: 'tag-update',\n        REJECTED: 'rejected',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Parse information from the given line as returned by git-fetch -v\n        and return a new FetchInfo object representing this information.\n        \"\"\"\n        match = cls._re_fetch_result.match(line)\n        if match is None:\n            raise ValueError(\"Failed to parse line: %r\" % line)\n\n        # parse lines\n        control_character, operation, local_remote_ref, remote_local_ref, note = \\\n            match.groups()\n\n        # parse flags from control_character\n        flags = 0\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\n                \"Control character %r unknown as parsed from line %r\"\n                % (control_character, line))\n        # END control char exception handling\n\n        # parse operation string for more info - makes no sense for symbolic refs,\n        # but we parse it anyway\n        old_commit = None\n        if 'rejected' in operation:\n            flags |= cls.REJECTED\n        if 'new tag' in operation:\n            flags |= cls.NEW_TAG\n        if 'tag update' in operation:\n            flags |= cls.TAG_UPDATE\n        if 'new branch' in operation:\n            flags |= cls.NEW_HEAD\n        if '...' in operation or '..' in operation:\n            split_token = '...'\n            if control_character == ' ':\n                split_token = split_token[:-1]\n            old_commit = operation.split(split_token)[0]\n        # END handle refspec\n\n        return cls(\n            ref=remote_local_ref.strip(),\n            local_ref=local_remote_ref.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=note,\n            old_commit=old_commit,\n        )\n\n\nclass PushInfo(dict):\n    \"\"\"dict that carries results of a push operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    NEW_TAG, NEW_HEAD, NO_MATCH, REJECTED, REMOTE_REJECTED, REMOTE_FAILURE, DELETED, \\\n        FORCED_UPDATE, FAST_FORWARD, UP_TO_DATE, ERROR = [1 << x for x in range(11)]\n\n    _flag_map = {'X': NO_MATCH,\n                 '-': DELETED,\n                 '*': 0,\n                 '+': FORCED_UPDATE,\n                 ' ': FAST_FORWARD,\n                 '=': UP_TO_DATE,\n                 '!': ERROR}\n\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        NO_MATCH: 'no-match',\n        REJECTED: 'rejected',\n        REMOTE_REJECTED: 'remote-rejected',\n        REMOTE_FAILURE: 'remote-failure',\n        DELETED: 'deleted',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        UP_TO_DATE: 'uptodate',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Create a new PushInfo instance as parsed from line which is expected to be like\n            refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes\"\"\"\n        control_character, from_to, summary = line.split('\\t', 3)\n        flags = 0\n\n        # control character handling\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\"Control character %r unknown as parsed from line %r\" % (control_character, line))\n        # END handle control character\n\n        # from_to handling\n        from_ref_string, to_ref_string = from_to.split(':')\n\n        # commit handling, could be message or commit info\n        old_commit = None\n        if summary.startswith('['):\n            if \"[rejected]\" in summary:\n                flags |= cls.REJECTED\n            elif \"[remote rejected]\" in summary:\n                flags |= cls.REMOTE_REJECTED\n            elif \"[remote failure]\" in summary:\n                flags |= cls.REMOTE_FAILURE\n            elif \"[no match]\" in summary:\n                flags |= cls.ERROR\n            elif \"[new tag]\" in summary:\n                flags |= cls.NEW_TAG\n            elif \"[new branch]\" in summary:\n                flags |= cls.NEW_HEAD\n            # uptodate encoded in control character\n        else:\n            # fast-forward or forced update - was encoded in control character,\n            # but we parse the old and new commit\n            split_token = \"...\"\n            if control_character == \" \":\n                split_token = \"..\"\n            old_sha, _new_sha = summary.split(' ')[0].split(split_token)\n            # have to use constructor here as the sha usually is abbreviated\n            old_commit = old_sha\n        # END message handling\n\n        return cls(\n            from_ref=from_ref_string.strip(),\n            to_ref=to_ref_string.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=summary.strip(),\n            old_commit=old_commit,\n        )\n\n\n# Compatibility kludge.  See GitRepo.get_submodules().\nSubmodule = namedtuple(\"Submodule\", [\"name\", \"path\", \"url\"])\n\n\nclass GitRepo(RepoInterface, metaclass=PathBasedFlyweight):\n    \"\"\"Representation of a git repository\n\n    \"\"\"\n    # We must check git config to have name and email set, but\n    # should do it once\n    _config_checked = False\n\n    # Begin Flyweight:\n\n    _unique_instances = WeakValueDictionary()\n\n    def _flyweight_invalid(self):\n        return not self.is_valid_git()\n\n    @classmethod\n    def _flyweight_reject(cls, id_, *args, **kwargs):\n        # TODO:\n        # This is a temporary approach. See PR # ...\n        # create = kwargs.pop('create', None)\n        # kwargs.pop('path', None)\n        # if create and kwargs:\n        #     # we have `create` plus options other than `path`\n        #     return \"Call to {0}() with args {1} and kwargs {2} conflicts \" \\\n        #            \"with existing instance {3}.\" \\\n        #            \"This is likely to be caused by inconsistent logic in \" \\\n        #            \"your code.\" \\\n        #            \"\".format(cls, args, kwargs, cls._unique_instances[id_])\n        pass\n\n    # End Flyweight\n\n    def __hash__(self):\n        # the flyweight key is already determining unique instances\n        # add the class name to distinguish from strings of a path\n        return hash((self.__class__.__name__, self.__weakref__.key))\n\n    # This is the least common denominator to claim that a user\n    # used DataLad.\n    # For now citing Zenodo's all (i.e., latest) version\n    @due.dcite(Doi(\"10.5281/zenodo.808846\"),\n               # override path since there is no need ATM for such details\n               path=\"datalad\",\n               description=\"DataLad - Data management and distribution platform\")\n    def __init__(self, path, url=None, runner=None, create=True,\n                 git_opts=None, repo=None, fake_dates=False,\n                 create_sanity_checks=True,\n                 **kwargs):\n        \"\"\"Creates representation of git repository at `path`.\n\n        Can also be used to create a git repository at `path`.\n\n        Parameters\n        ----------\n        path: str\n          path to the git repository; In case it's not an absolute path,\n          it's relative to PWD\n        url: str, optional\n          DEPRECATED -- use .clone() class method\n          url to the to-be-cloned repository. Requires a valid git url\n          according to:\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        create: bool, optional\n          if true, creates a git repository at `path` if there is none. Also\n          creates `path`, if it doesn't exist.\n          If set to false, an exception is raised in case `path` doesn't exist\n          or doesn't contain a git repository.\n        repo: git.Repo, optional\n          GitPython's Repo instance to (re)use if provided\n        create_sanity_checks: bool, optional\n          Whether to perform sanity checks during initialization (when\n          `create=True` and target path is not a valid repo already), such as\n          that new repository is not created in the directory where git already\n          tracks some files.\n        kwargs:\n          keyword arguments serving as additional options to the git-init\n          command. Therefore, it makes sense only if called with `create`.\n\n          Generally, this way of passing options to the git executable is\n          (or will be) used a lot in this class. It's a transformation of\n          python-style keyword arguments (or a `dict`) to command line arguments,\n          provided by GitPython.\n\n          A single character keyword will be prefixed by '-', multiple characters\n          by '--'. An underscore in the keyword becomes a dash. The value of the\n          keyword argument is used as the value for the corresponding command\n          line argument. Assigning a boolean creates a flag.\n\n          Examples:\n          no_commit=True => --no-commit\n          C='/my/path'   => -C /my/path\n\n        \"\"\"\n\n        # BEGIN Repo validity test\n        # We want to fail early for tests, that would be performed a lot. In particular this is about\n        # GitRepo.is_valid_repo. We would use the latter to decide whether or not to call GitRepo() only for\n        # __init__ to then test the same things again. If we fail early we can save the additional test from outer\n        # scope.\n        self.path = path\n\n        # Note, that the following three path objects are used often and therefore\n        # are stored for performance. Path object creation comes with a cost. Most noteably,\n        # this is used for validity checking of the repository.\n        self.pathobj = ut.Path(self.path)\n        self.dot_git = self.pathobj / '.git'\n        # Read a potential .git file in order to not do that over and over again, when testing is_valid_git() etc.\n        # TODO: There's still some code duplication with static method GitRepo.get_git_dir()\n        #       However, it's returning relative path. So, the logic in usage needs to be unified in order to melt both\n        #       pieces.\n        if self.dot_git.is_file():\n            with self.dot_git.open() as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    self.dot_git = self.pathobj / line[7:].strip()\n                else:\n                    raise InvalidGitRepositoryError(\"Invalid .git file\")\n\n        elif self.dot_git.is_symlink():\n            self.dot_git = self.dot_git.resolve()\n\n        self._valid_git_test_path = self.dot_git / 'HEAD'\n        _valid_repo = self.is_valid_git()\n\n        do_create = False\n        if create and not _valid_repo:\n            if repo is not None:\n                # `repo` passed with `create`, which doesn't make sense\n                raise TypeError(\"argument 'repo' must not be used with 'create'\")\n            do_create = True\n        else:\n            # Note: We used to call gitpy.Repo(path) here, which potentially\n            # raised NoSuchPathError or InvalidGitRepositoryError. This is\n            # used by callers of GitRepo.__init__() to detect whether we have a\n            # valid repo at `path`. Now, with switching to lazy loading property\n            # `repo`, we detect those cases without instantiating a\n            # gitpy.Repo().\n\n            if not exists(path):\n                raise NoSuchPathError(path)\n            if not _valid_repo:\n                raise InvalidGitRepositoryError(path)\n        # END Repo validity test\n\n        if url is not None:\n            raise DeprecatedError(\n                new=\".clone() class method\",\n                version=\"0.5.0\",\n                msg=\"RF: url passed to init()\"\n            )\n\n        # So that we \"share\" control paths with git/git-annex\n        if ssh_manager:\n            ssh_manager.assure_initialized()\n\n        # note: we may also want to distinguish between a path to the worktree\n        # and the actual repository\n\n        # Could be used to e.g. disable automatic garbage and autopacking\n        # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0']\n        self._GIT_COMMON_OPTIONS = []\n        # actually no need with default GitPython db backend not in memory\n        # default_git_odbt but still allows for faster testing etc.\n        # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = []\n\n        if git_opts is None:\n            git_opts = {}\n        if kwargs:\n            git_opts.update(kwargs)\n\n        self.cmd_call_wrapper = runner or GitRunner(cwd=self.path)\n        self._repo = repo\n        self._cfg = None\n\n        if do_create:  # we figured it out ealier\n            self._repo = self._create_empty_repo(path,\n                                                 create_sanity_checks, **git_opts)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # calls above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        # with DryRunProtocol path might still not exist\n        if exists(self.path):\n            self.inode = os.stat(self.path).st_ino\n        else:\n            self.inode = None\n\n        if fake_dates:\n            self.configure_fake_dates()\n        # Set by fake_dates_enabled to cache config value across this instance.\n        self._fake_dates_enabled = None\n\n    def _create_empty_repo(self, path, sanity_checks=True, **kwargs):\n        if not op.lexists(path):\n            os.makedirs(path)\n        elif sanity_checks and external_versions['cmd:git'] < '2.14.0':\n            warnings.warn(\n                \"Your git version (%s) is too old, we will not safe-guard \"\n                \"against creating a new repository under already known to git \"\n                \"subdirectory\" % external_versions['cmd:git'],\n                OutdatedExternalDependencyWarning\n            )\n        elif sanity_checks:\n            # Verify that we are not trying to initialize a new git repository\n            # under a directory some files of which are already tracked by git\n            # use case: https://github.com/datalad/datalad/issues/3068\n            try:\n                stdout, _ = self._git_custom_command(\n                    None, ['git', 'ls-files'], cwd=path, expect_fail=True\n                )\n                if stdout:\n                    raise PathKnownToRepositoryError(\n                        \"Failing to initialize new repository under %s where \"\n                        \"following files are known to a repository above: %s\"\n                        % (path, stdout)\n                    )\n            except CommandError:\n                # assume that all is good -- we are not under any repo\n                pass\n\n        cmd = ['git', 'init']\n        cmd.extend(kwargs.pop('_from_cmdline_', []))\n        cmd.extend(to_options(**kwargs))\n        lgr.debug(\n            \"Initialize empty Git repository at '%s'%s\",\n            path,\n            ' %s' % cmd[2:] if cmd[2:] else '')\n\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                cmd,\n                cwd=path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            lgr.error(exc_str(exc))\n            raise\n        # we want to return None and have lazy eval take care of\n        # the rest\n        return\n\n    @property\n    def repo(self):\n        # with DryRunProtocol path not exist\n        if exists(self.path):\n            inode = os.stat(self.path).st_ino\n        else:\n            inode = None\n        if self.inode != inode:\n\n            # TODO: - what if we never had any inode? possible?\n            #       - also: call to self._repo and only afterwards checking whether it's None seems strange\n\n            # reset background processes invoked by GitPython:\n            # it can be that we never got to instantiate a GitPython repo instance\n            if self._repo:\n                self._repo.git.clear_cache()\n            self.inode = inode\n\n        if self._repo is None:\n            # Note, that this may raise GitCommandError, NoSuchPathError,\n            # InvalidGitRepositoryError:\n            self._repo = self.cmd_call_wrapper(\n                Repo,\n                self.path)\n            lgr.log(8, \"Using existing Git repository at %s\", self.path)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # call of Repo(path) above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        return self._repo\n\n    @classmethod\n    def clone(cls, url, path, *args, clone_options=None, **kwargs):\n        \"\"\"Clone url into path\n\n        Provides workarounds for known issues (e.g.\n        https://github.com/datalad/datalad/issues/785)\n\n        Parameters\n        ----------\n        url : str\n        path : str\n        clone_options : dict\n          Key/value pairs of arbitrary options that will be passed on to the\n          underlying call to `git-clone`.\n        expect_fail : bool\n          Whether expect that command might fail, so error should be logged then\n          at DEBUG level instead of ERROR\n        \"\"\"\n\n        if 'repo' in kwargs:\n            raise TypeError(\"argument 'repo' conflicts with cloning\")\n            # TODO: what about 'create'?\n\n        expect_fail = kwargs.pop('expect_fail', False)\n        # fail early on non-empty target:\n        from os import listdir\n        if exists(path) and listdir(path):\n            # simulate actual GitCommandError:\n            lgr.warning(\"destination path '%s' already exists and is not an \"\n                        \"empty directory.\" % path)\n            raise GitCommandError(\n                ['git', 'clone', '-v', url, path],\n                128,\n                \"fatal: destination path '%s' already exists and is not an \"\n                \"empty directory.\" % path)\n        else:\n            # protect against cloning into existing and obviously dangling\n            # instance for that location\n            try:\n                del cls._unique_instances[path]\n            except KeyError:\n                # didn't exist - all fine\n                pass\n\n        # Massage URL\n        url_ri = RI(url) if not isinstance(url, RI) else url\n        if not on_windows:\n            # if we are on windows, the local path of a URL\n            # would not end up being a proper local path and cloning\n            # would fail. Don't try to be smart and just pass the\n            # URL along unmodified\n\n            # try to get a local path from `url`:\n            try:\n                url = url_ri.localpath\n                url_ri = RI(url)\n            except ValueError:\n                pass\n\n        if is_ssh(url_ri):\n            ssh_manager.get_connection(url).open()\n        else:\n            if isinstance(url_ri, PathRI):\n                # expand user, because execution not going through a shell\n                # doesn't work well otherwise\n                new_url = os.path.expanduser(url)\n                if url != new_url:\n                    lgr.info(\"Expanded source path to %s from %s\", new_url, url)\n                    url = new_url\n\n        fix_annex = None\n        ntries = 5  # 3 is not enough for robust workaround\n        for trial in range(ntries):\n            try:\n                lgr.debug(\"Git clone from {0} to {1}\".format(url, path))\n\n                out, err = WitlessRunner(\n                    env=GitRunner.get_git_environ_adjusted()).run(\n                        ['git', 'clone', '--progress', url, path] \\\n                        + (to_options(**clone_options)\n                           if clone_options else []),\n                        protocol=GitProgress,\n                )\n                # fish out non-critical warnings by git-clone\n                # (empty repo clone, etc.), all other content is logged\n                # by the progress helper to 'debug'\n                for errline in err.splitlines():\n                    if errline.startswith('warning:'):\n                        lgr.warning(errline[8:].strip())\n                lgr.debug(\"Git clone completed\")\n                break\n            except CommandError as e:\n                # log here but let caller decide what to do\n                e_str = exc_str(e)\n                # see https://github.com/datalad/datalad/issues/785\n                if re.search(\"Request for .*aborted.*Unable to find\", str(e),\n                             re.DOTALL) \\\n                        and trial < ntries - 1:\n                    lgr.info(\n                        \"Hit a known issue with Git (see GH#785). Trial #%d, \"\n                        \"retrying\",\n                        trial)\n                    continue\n                    (lgr.debug if expect_fail else lgr.error)(e_str)\n\n                if \"Clone succeeded, but checkout failed.\" in str(e):\n                    fix_annex = e\n                    break\n\n                raise\n\n        # get ourselves a repository instance\n        gr = cls(path, *args, **kwargs)\n        if fix_annex:\n            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then\n            # would need to import our own subclass\n            if hasattr(gr, 'is_valid_annex'):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                if not gr.is_initialized():\n                    gr._init()\n                gr.fsck()\n            else:\n                lgr.warning(\"Experienced issues while cloning: %s\", exc_str(fix_annex))\n        return gr\n\n    def __del__(self):\n        # unbind possibly bound ConfigManager, to prevent all kinds of weird\n        # stalls etc\n        self._cfg = None\n        # Make sure to flush pending changes, especially close batch processes\n        # (internal `git cat-file --batch` by GitPython)\n        try:\n            if getattr(self, '_repo', None) is not None and exists(self.path):\n                # gc might be late, so the (temporary)\n                # repo doesn't exist on FS anymore\n                self._repo.git.clear_cache()\n                # We used to write out the index to flush GitPython's\n                # state... but such unconditional write is really a workaround\n                # and does not play nice with read-only operations - permission\n                # denied etc. So disabled \n                #if exists(opj(self.path, '.git')):  # don't try to write otherwise\n                #    self.repo.index.write()\n        except (InvalidGitRepositoryError, AttributeError):\n            # might have being removed and no longer valid or attributes unbound\n            pass\n\n    def __repr__(self):\n        return \"<GitRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def __eq__(self, obj):\n        \"\"\"Decides whether or not two instances of this class are equal.\n\n        This is done by comparing the base repository path.\n        \"\"\"\n        return self.path == obj.path\n\n    def is_valid_git(self):\n        \"\"\"Returns whether the underlying repository appears to be still valid\n\n        Note, that this almost identical to the classmethod is_valid_repo(). However,\n        if we are testing an existing instance, we can save Path object creations. Since this testing\n        is done a lot, this is relevant. Creation of the Path objects in is_valid_repo() takes nearly half the time of\n        the entire function.\n\n        Also note, that this method is bound to an instance but still class-dependent, meaning that a subclass\n        cannot simply overwrite it. This is particularly important for the call from within __init__(),\n        which in turn is called by the subclasses' __init__. Using an overwrite would lead to the wrong thing being\n        called.\n        \"\"\"\n\n        return self.dot_git.exists() and (\n                not self.dot_git.is_dir() or self._valid_git_test_path.exists()\n        )\n\n    @classmethod\n    def is_valid_repo(cls, path):\n        \"\"\"Returns if a given path points to a git repository\"\"\"\n        if not isinstance(path, Path):\n            path = Path(path)\n        dot_git_path = path / '.git'\n\n        # the aim here is to have this test as cheap as possible, because\n        # it is performed a lot\n        # recognize two things as good-enough indicators of a present\n        # repo: 1) a non-empty .git directory (#3473)\n        #          NOTE: It's actually faster (and more accurate) to test for existence of a particular subpath.\n        #                This should be something that's there right after git-init. Going for .git/HEAD ATM.\n        #\n        #                In [11]: %timeit path.exists() and (not path.is_dir() or head_path.exists())\n        #                4.93 µs ± 34.8 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n        #                In [12]: %timeit path.exists() and (not path.is_dir() or any(path.iterdir()))\n        #                12.8 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n        #\n        #   and 2) a pointer file or symlink\n\n        return dot_git_path.exists() and (\n            not dot_git_path.is_dir() or (dot_git_path / 'HEAD').exists()\n        )\n\n    @staticmethod\n    def get_git_dir(repo):\n        \"\"\"figure out a repo's gitdir\n\n        '.git' might be a  directory, a symlink or a file\n\n        Note\n        ----\n        Please try using GitRepo.dot_git instead! That one's not static, but it's cheaper and you should avoid\n        not having an instance of a repo you're working on anyway. Note, that the property in opposition to this method\n        returns an absolute path.\n\n\n        Parameters\n        ----------\n        repo: path or Repo instance\n          currently expected to be the repos base dir\n\n        Returns\n        -------\n        str\n          relative path to the repo's git dir; So, default would be \".git\"\n        \"\"\"\n        if hasattr(repo, 'path'):\n            # repo instance like given\n            repo = repo.path\n        dot_git = op.join(repo, \".git\")\n        if not op.exists(dot_git):\n            raise RuntimeError(\"Missing .git in %s.\" % repo)\n        elif op.islink(dot_git):\n            git_dir = os.readlink(dot_git)\n        elif op.isdir(dot_git):\n            git_dir = \".git\"\n        elif op.isfile(dot_git):\n            with open(dot_git) as f:\n                git_dir = f.readline()\n                if git_dir.startswith(\"gitdir:\"):\n                    git_dir = git_dir[7:]\n                git_dir = git_dir.strip()\n\n        return git_dir\n\n    @property\n    def config(self):\n        \"\"\"Get an instance of the parser for the persistent repository\n        configuration.\n\n        Note: This allows to also read/write .datalad/config,\n        not just .git/config\n\n        Returns\n        -------\n        ConfigManager\n        \"\"\"\n        if self._cfg is None:\n            # associate with this dataset and read the entire config hierarchy\n            self._cfg = ConfigManager(dataset=self, source='any')\n        return self._cfg\n\n    def is_with_annex(self):\n        \"\"\"Report if GitRepo (assumed) has (remotes with) a git-annex branch\n        \"\"\"\n        return any(\n            b['refname:strip=2'] == 'git-annex' or b['refname:strip=2'].endswith('/git-annex')\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern=['refs/heads', 'refs/remotes'])\n        )\n\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n        cmd = ['git']\n        if git_options:\n            cmd.extend(git_options)\n        cmd += [\"rev-parse\", \"--show-toplevel\"]\n        try:\n            toppath, err = GitRunner().run(\n                cmd,\n                cwd=path,\n                log_stdout=True, log_stderr=True,\n                expect_fail=True, expect_stderr=True)\n            toppath = toppath.rstrip('\\n\\r')\n        except CommandError:\n            return None\n        except OSError:\n            toppath = GitRepo.get_toppath(dirname(path), follow_up=follow_up,\n                                          git_options=git_options)\n\n        if follow_up:\n            path_ = path\n            path_prev = \"\"\n            while path_ and path_ != path_prev:  # on top /.. = /\n                if realpath(path_) == toppath:\n                    toppath = path_\n                    break\n                path_prev = path_\n                path_ = dirname(path_)\n\n        return toppath\n\n    @normalize_paths\n    def add(self, files, git=True, git_options=None, update=False):\n        \"\"\"Adds file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list\n          list of paths to add\n        git: bool\n          somewhat ugly construction to be compatible with AnnexRepo.add();\n          has to be always true.\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n        Returns\n        -------\n        list\n          Of status dicts.\n        \"\"\"\n        # under all circumstances call this class' add_ (otherwise\n        # AnnexRepo.add would go into a loop\n        return list(GitRepo.add_(self, files, git=git, git_options=git_options,\n                    update=update))\n\n    def add_(self, files, git=True, git_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        # TODO: git_options is used as options for the git-add here,\n        # instead of options to the git executable => rename for consistency\n\n        if not git:\n            lgr.warning(\n                'GitRepo.add() called with git=%s, this should not happen',\n                git)\n            git = True\n\n        # there is no other way then to collect all files into a list\n        # at this point, because we need to pass them at once to a single\n        # `git add` call\n        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]\n\n        if not (files or git_options or update):\n            # wondering why just a warning? in cmdline this is also not an error\n            lgr.warning(\"add was called with empty file list and no options.\")\n            return\n\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                files,\n                # Set annex.largefiles to prevent storing files in annex when\n                # GitRepo() is instantiated with a v6+ annex repo.\n                ['git', '-c', 'annex.largefiles=nothing', 'add'] +\n                ensure_list(git_options) +\n                to_options(update=update) + ['--verbose']\n            )\n            # get all the entries\n            for o in self._process_git_get_output(*add_out):\n                yield o\n            # Note: as opposed to git cmdline, force is True by default in\n            #       gitpython, which would lead to add things, that are\n            #       ignored or excluded otherwise\n            # 2. Note: There is an issue with globbing (like adding '.'),\n            #       which apparently doesn't care for 'force' and therefore\n            #       adds '.git/...'. May be it's expanded at the wrong\n            #       point in time or sth. like that.\n            # For now, use direct call to git add.\n            #self.cmd_call_wrapper(self.repo.index.add, files, write=True,\n            #                      force=False)\n            # TODO: May be make use of 'fprogress'-option to indicate\n            # progress\n            # But then, we don't have it for git-annex add, anyway.\n            #\n            # TODO: Is write=True a reasonable way to do it?\n            # May be should not write until success of operation is\n            # confirmed?\n            # What's best in case of a list of files?\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n        # Make sure return value from GitRepo is consistent with AnnexRepo\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return\n\n    @staticmethod\n    def _process_git_get_output(stdout, stderr=None):\n        \"\"\"Given both outputs (stderr is ignored atm) of git add - process it\n\n        Primarily to centralize handling in both indirect annex and direct\n        modes when ran through proxy\n        \"\"\"\n        return [{u'file': f, u'success': True}\n                for f in re.findall(\"'(.*)'[\\n$]\", ensure_unicode(stdout))]\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, recursive=False, **kwargs):\n        \"\"\"Remove files.\n\n        Calls git-rm.\n\n        Parameters\n        ----------\n        files: str\n          list of paths to remove\n        recursive: False\n          whether to allow recursive removal from subdirectories\n        kwargs:\n          see `__init__`\n\n        Returns\n        -------\n        [str]\n          list of successfully removed files.\n        \"\"\"\n\n        files = _remove_empty_items(files)\n\n        if recursive:\n            kwargs['r'] = True\n        stdout, stderr = self._git_custom_command(\n            files, ['git', 'rm'] + to_options(**kwargs))\n\n        # output per removed file is expected to be \"rm 'PATH'\":\n        return [line.strip()[4:-1] for line in stdout.splitlines()]\n\n        #return self.repo.git.rm(files, cached=False, **kwargs)\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks\n        \"\"\"\n        # All GitPython commands should take care about flushing index\n        # whenever they modify it, so we would not care to do anything\n        # if self.repo is not None and exists(opj(self.path, '.git')):  # don't try to write otherwise:\n        #     # flush possibly cached in GitPython changes to index:\n        #     # if self.repo.git:\n        #     #     sys.stderr.write(\"CLEARING\\n\")\n        #     #     self.repo.git.clear_cache()\n        #     self.repo.index.write()\n\n        # Close batched by GitPython git processes etc\n        # Ref: https://github.com/gitpython-developers/GitPython/issues/718\n        self.repo.__del__()\n        pass\n\n    @staticmethod\n    def _get_prefixed_commit_msg(msg):\n        DATALAD_PREFIX = \"[DATALAD]\"\n        return DATALAD_PREFIX if not msg else \"%s %s\" % (DATALAD_PREFIX, msg)\n\n    def for_each_ref_(self, fields=('objectname', 'objecttype', 'refname'),\n                      pattern=None, points_at=None, sort=None, count=None,\n                      contains=None):\n        \"\"\"Wrapper for `git for-each-ref`\n\n        Please see manual page git-for-each-ref(1) for a complete overview\n        of its functionality. Only a subset of it is supported by this\n        wrapper.\n\n        Parameters\n        ----------\n        fields : iterable or str\n          Used to compose a NULL-delimited specification for for-each-ref's\n          --format option. The default field list reflects the standard\n          behavior of for-each-ref when the --format option is not given.\n        pattern : list or str, optional\n          If provided, report only refs that match at least one of the given\n          patterns.\n        points_at : str, optional\n          Only list refs which points at the given object.\n        sort : list or str, optional\n          Field name(s) to sort-by. If multiple fields are given, the last one\n          becomes the primary key. Prefix any field name with '-' to sort in\n          descending order.\n        count : int, optional\n          Stop iteration after the given number of matches.\n        contains : str, optional\n          Only list refs which contain the specified commit.\n\n        Yields\n        ------\n        dict with items matching the given `fields`\n\n        Raises\n        ------\n        ValueError\n          if no `fields` are given\n\n        RuntimeError\n          if `git for-each-ref` returns a record where the number of\n          properties does not match the number of `fields`\n        \"\"\"\n        if not fields:\n            raise ValueError('no `fields` provided, refuse to proceed')\n        fields = ensure_list(fields)\n        cmd = [\n            \"git\",\n            \"for-each-ref\",\n            \"--format={}\".format(\n                '%00'.join(\n                    '%({})'.format(f) for f in fields)),\n        ]\n        if points_at:\n            cmd.append('--points-at={}'.format(points_at))\n        if contains:\n            cmd.append('--contains={}'.format(contains))\n        if sort:\n            for k in ensure_list(sort):\n                cmd.append('--sort={}'.format(k))\n        if pattern:\n            cmd += ensure_list(pattern)\n        if count:\n            cmd.append('--count={:d}'.format(count))\n\n        out, _ = self._git_custom_command(None, cmd)\n        for line in out.splitlines():\n            props = line.split('\\0')\n            if len(fields) != len(props):\n                raise RuntimeError(\n                    'expected fields {} from git-for-each-ref, but got: {}'.format(\n                        fields, props))\n            yield dict(zip(fields, props))\n\n    def configure_fake_dates(self):\n        \"\"\"Configure repository to use fake dates.\n        \"\"\"\n        lgr.debug(\"Enabling fake dates\")\n        self.config.set(\"datalad.fake-dates\", \"true\")\n\n    @property\n    def fake_dates_enabled(self):\n        \"\"\"Is the repository configured to use fake dates?\n        \"\"\"\n        if self._fake_dates_enabled is None:\n            self._fake_dates_enabled = \\\n                self.config.getbool('datalad', 'fake-dates', default=False)\n        return self._fake_dates_enabled\n\n    def add_fake_dates(self, env):\n        \"\"\"Add fake dates to `env`.\n\n        Parameters\n        ----------\n        env : dict or None\n            Environment variables.\n\n        Returns\n        -------\n        A dict (copied from env), with date-related environment\n        variables for git and git-annex set.\n        \"\"\"\n        env = (env if env is not None else os.environ).copy()\n        # Note: Use _git_custom_command here rather than repo.git.for_each_ref\n        # so that we use annex-proxy in direct mode.\n        last_date = list(self.for_each_ref_(\n            fields='committerdate:raw',\n            count=1,\n            pattern='refs/heads',\n            sort=\"-committerdate\",\n        ))\n\n        if last_date:\n            # Drop the \"contextual\" timezone, leaving the unix timestamp.  We\n            # avoid :unix above because it wasn't introduced until Git v2.9.4.\n            last_date = last_date[0]['committerdate:raw'].split()[0]\n            seconds = int(last_date)\n        else:\n            seconds = self.config.obtain(\"datalad.fake-dates-start\")\n        seconds_new = seconds + 1\n        date = \"@{} +0000\".format(seconds_new)\n\n        lgr.debug(\"Setting date to %s\",\n                  time.strftime(\"%a %d %b %Y %H:%M:%S +0000\",\n                                time.gmtime(seconds_new)))\n\n        env[\"GIT_AUTHOR_DATE\"] = date\n        env[\"GIT_COMMITTER_DATE\"] = date\n        env[\"GIT_ANNEX_VECTOR_CLOCK\"] = str(seconds_new)\n\n        return env\n\n    def commit(self, msg=None, options=None, _datalad_msg=False, careless=True,\n               files=None, date=None, index_file=None):\n        \"\"\"Commit changes to git.\n\n        Parameters\n        ----------\n        msg: str, optional\n          commit-message\n        options: list of str, optional\n          cmdline options for git-commit\n        _datalad_msg: bool, optional\n          To signal that commit is automated commit by datalad, so\n          it would carry the [DATALAD] prefix\n        careless: bool, optional\n          if False, raise when there's nothing actually committed;\n          if True, don't care\n        files: list of str, optional\n          path(s) to commit\n        date: str, optional\n          Date in one of the formats git understands\n        index_file: str, optional\n          An alternative index to use\n        \"\"\"\n\n        self.precommit()\n\n        # assemble commandline\n        cmd = ['git', 'commit']\n        options = ensure_list(options)\n\n        if date:\n            options += [\"--date\", date]\n\n        if not msg:\n            msg = 'Recorded changes'\n            _datalad_msg = True\n\n        if _datalad_msg:\n            msg = self._get_prefixed_commit_msg(msg)\n\n        options += [\"-m\", msg]\n        cmd.extend(options)\n\n        # set up env for commit\n        env = GitRunner.get_git_environ_adjusted()\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n        if index_file:\n            env['GIT_INDEX_FILE'] = index_file\n\n        lgr.debug(\"Committing via direct call of git: %s\" % cmd)\n\n        file_chunks = generate_file_chunks(files, cmd) if files else [[]]\n\n        runner = WitlessRunner(cwd=self.path, env=env)\n\n        try:\n            for i, chunk in enumerate(file_chunks):\n                cur_cmd = cmd + (\n                    # if this is an explicit dry-run, there is no point in\n                    # amending, because no commit was ever made\n                    # otherwise, amend the first commit, and prevent\n                    # leaving multiple commits behind\n                    ['--amend', '--no-edit']\n                    if i > 0 and '--dry-run' not in cmd\n                    else []\n                ) + ['--'] + chunk\n                runner.run(\n                    cur_cmd,\n                    protocol=StdOutErrCapture,\n                    stdin=None,\n                )\n        except CommandError as e:\n            # real errors first\n            if \"did not match any file(s) known to git\" in e.stderr:\n                raise FileNotInRepositoryError(\n                    cmd=e.cmd,\n                    msg=\"File(s) unknown to git\",\n                    code=e.code,\n                    filename=linesep.join([\n                        l for l in e.stderr.splitlines()\n                        if l.startswith(\"error: pathspec\")\n                    ])\n                )\n            # behavior choices now\n            elif not careless:\n                # not willing to compromise at all\n                raise\n            elif 'nothing to commit' in e.stdout:\n                lgr.debug(\"nothing to commit in %s. Ignored.\", self)\n            elif 'no changes added to commit' in e.stdout or \\\n                    'nothing added to commit' in e.stdout:\n                lgr.debug(\"no changes added to commit in %s. Ignored.\", self)\n            else:\n                raise\n\n    # TODO usage is primarily in the tests, consider making a test helper and\n    # remove from GitRepo API\n    def get_indexed_files(self):\n        \"\"\"Get a list of files in git's index\n\n        Returns\n        -------\n        list\n            list of paths rooting in git's base dir\n        \"\"\"\n\n        return [\n            str(r.relative_to(self.pathobj))\n            for r in self.get_content_info(\n                paths=None, ref=None, untracked='no', eval_file_type=False)\n        ]\n\n    def format_commit(self, fmt, commitish=None):\n        \"\"\"Return `git show` output for `commitish`.\n\n        Parameters\n        ----------\n        fmt : str\n            A format string accepted by `git show`.\n        commitish: str, optional\n          Any commit identifier (defaults to \"HEAD\").\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        cmd = ['git', 'show', '-z', '--no-patch', '--format=' + fmt]\n        if commitish is not None:\n            cmd.append(commitish + \"^{commit}\")\n        # make sure Git takes our argument as a revision\n        cmd.append('--')\n        try:\n            stdout, stderr = self._git_custom_command(\n                '', cmd, expect_stderr=True, expect_fail=True)\n        except CommandError as e:\n            if 'bad revision' in e.stderr:\n                raise ValueError(\"Unknown commit identifier: %s\" % commitish)\n            elif 'does not have any commits yet' in e.stderr:\n                return None\n            else:\n                raise e\n        # This trailing null is coming from the -z above, which avoids the\n        # newline that Git would append to the output. We could drop -z and\n        # strip the newline directly, but then we'd have to worry about\n        # compatibility across platforms.\n        return stdout.rsplit(\"\\0\", 1)[0]\n\n    def get_hexsha(self, commitish=None, short=False):\n        \"\"\"Return a hexsha for a given commitish.\n\n        Parameters\n        ----------\n        commitish : str, optional\n          Any identifier that refers to a commit (defaults to \"HEAD\").\n        short : bool, optional\n          Return the abbreviated form of the hexsha.\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        stdout = self.format_commit(\"%{}\".format('h' if short else 'H'),\n                                    commitish)\n        if stdout is not None:\n            stdout = stdout.splitlines()\n            assert(len(stdout) == 1)\n            return stdout[0]\n\n    @normalize_paths(match_return_type=False)\n    def get_last_commit_hexsha(self, files):\n        \"\"\"Return the hash of the last commit the modified any of the given\n        paths\"\"\"\n        try:\n            commit = self.call_git(\n                ['rev-list', '-n1', 'HEAD'],\n                files=files,\n                expect_fail=True,\n            )\n            commit = commit.strip()\n            return commit if commit else None\n        except CommandError:\n            if self.get_hexsha() is None:\n                # unborn branch, don't freak out\n                return None\n            raise\n\n    def get_revisions(self, revrange=None, fmt=\"%H\", options=None):\n        \"\"\"Return list of revisions in `revrange`.\n\n        Parameters\n        ----------\n        revrange : str or list of str or None, optional\n            Revisions or revision ranges to walk. If None, revision defaults to\n            HEAD unless a revision-modifying option like `--all` or\n            `--branches` is included in `options`.\n        fmt : string, optional\n            Format accepted by `--format` option of `git log`. This should not\n            contain new lines because the output is split on new lines.\n        options : list of str, optional\n            Options to pass to `git log`.  This should not include `--format`.\n\n        Returns\n        -------\n        List of revisions (str), formatted according to `fmt`.\n        \"\"\"\n        if revrange is None:\n            revrange = []\n        elif isinstance(revrange, str):\n            revrange = [revrange]\n\n        cmd = [\"git\", \"log\", \"--format={}\".format(fmt)]\n        cmd.extend((options or []) + revrange + [\"--\"])\n        try:\n            stdout, _ = self._git_custom_command(None, cmd, expect_fail=True)\n        except CommandError as e:\n            if \"does not have any commits\" in e.stderr:\n                return []\n            raise\n        return stdout.splitlines()\n\n    def commit_exists(self, commitish):\n        \"\"\"Does `commitish` exist in the repo?\n\n        Parameters\n        ----------\n        commitish : str\n            A commit or an object that can be dereferenced to one.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        # Note: The peeling operator \"^{commit}\" is required so that rev-parse\n        # doesn't succeed if passed a full hexsha that is valid but doesn't\n        # exist.\n        return self.call_git_success(\n            [\"rev-parse\", \"--verify\", commitish + \"^{commit}\"])\n\n    def get_merge_base(self, commitishes):\n        \"\"\"Get a merge base hexsha\n\n        Parameters\n        ----------\n        commitishes: str or list of str\n          List of commitishes (branches, hexshas, etc) to determine the merge\n          base of. If a single value provided, returns merge_base with the\n          current branch.\n\n        Returns\n        -------\n        str or None\n          If no merge-base for given commits, or specified treeish doesn't\n          exist, None returned\n        \"\"\"\n        if isinstance(commitishes, str):\n            commitishes = [commitishes]\n        if not commitishes:\n            raise ValueError(\"Provide at least a single value\")\n        elif len(commitishes) == 1:\n            commitishes = commitishes + [self.get_active_branch()]\n\n        try:\n            base = self.call_git_oneline(['merge-base'] + commitishes)\n        except CommandError as exc:\n            if exc.code == 1 and not (exc.stdout or exc.stderr):\n                # No merge base was found (unrelated commits).\n                return None\n            if \"fatal: Not a valid object name\" in exc.stderr:\n                return None\n            raise\n\n        return base\n\n    def is_ancestor(self, reva, revb):\n        \"\"\"Is `reva` an ancestor of `revb`?\n\n        Parameters\n        ----------\n        reva, revb : str\n            Revisions.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return self.call_git_success(\n            [\"merge-base\", \"--is-ancestor\", reva, revb])\n\n    def get_commit_date(self, branch=None, date='authored'):\n        \"\"\"Get the date stamp of the last commit (in a branch or head otherwise)\n\n        Parameters\n        ----------\n        date: {'authored', 'committed'}\n          Which date to return.  \"authored\" will be the date shown by \"git show\"\n          and the one possibly specified via --date to `git commit`\n\n        Returns\n        -------\n        int or None\n          None if no commit\n        \"\"\"\n        if date == 'committed':\n            format = '%ct'\n        elif date == 'authored':\n            format = '%at'\n        else:\n            raise ValueError('unknow date type: {}'.format(date))\n        d = self.format_commit(format, commitish=branch)\n        return int(d) if d else None\n\n    def get_active_branch(self):\n        \"\"\"Get the name of the active branch\n\n        Returns\n        -------\n        str or None\n          Returns None if there is no active branch, i.e. detached HEAD,\n          and the branch name otherwise.\n        \"\"\"\n        try:\n            out, _ = self._git_custom_command(\n                \"\", [\"git\", \"symbolic-ref\", \"HEAD\"],\n                expect_fail=True)\n        except CommandError as e:\n            if 'HEAD is not a symbolic ref' in e.stderr:\n                lgr.debug(\"detached HEAD in {0}\".format(self))\n                return None\n            else:\n                raise e\n        return out.strip()[11:]  # strip refs/heads/\n\n    def get_branches(self):\n        \"\"\"Get all branches of the repo.\n\n        Returns\n        -------\n        [str]\n            Names of all branches of this repository.\n        \"\"\"\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/heads')\n        ]\n\n    def get_remote_branches(self):\n        \"\"\"Get all branches of all remotes of the repo.\n\n        Returns\n        -----------\n        [str]\n            Names of all remote branches.\n        \"\"\"\n        # TODO: Reconsider melting with get_branches()\n\n        # TODO: treat entries like this: origin/HEAD -> origin/master'\n        # currently this is done in collection\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/remotes')\n        ]\n\n    def get_remotes(self, with_urls_only=False):\n        \"\"\"Get known remotes of the repository\n\n        Parameters\n        ----------\n        with_urls_only : bool, optional\n          return only remotes which have urls\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n\n        # Note: read directly from config and spare instantiation of gitpy.Repo\n        # since we need this in AnnexRepo constructor. Furthermore gitpy does it\n        # pretty much the same way and the use of a Repo instance seems to have\n        # no reason other than a nice object oriented look.\n        from datalad.utils import unique\n\n        self.config.reload()\n        remotes = unique([x[7:] for x in self.config.sections()\n                          if x.startswith(\"remote.\")])\n\n        if with_urls_only:\n            remotes = [\n                r for r in remotes\n                if self.config.get('remote.%s.url' % r)\n            ]\n        return remotes\n\n    # TODO this is practically unused outside the tests, consider turning\n    # into a test helper and trim from the API\n    def get_files(self, branch=None):\n        \"\"\"Get a list of files in git.\n\n        Lists the files in the (remote) branch.\n\n        Parameters\n        ----------\n        branch: str\n          Name of the branch to query. Default: active branch.\n\n        Returns\n        -------\n        [str]\n          list of files.\n        \"\"\"\n        return [\n            str(p.relative_to(self.pathobj))\n            for p in self.get_content_info(\n                paths=None, ref=branch, untracked='no', eval_file_type=False)\n            ]\n\n    @normalize_paths(match_return_type=False)\n    def _git_custom_command(self, files, cmd_str,\n                            log_stdout=True, log_stderr=True, log_online=False,\n                            expect_stderr=True, cwd=None, env=None,\n                            shell=None, expect_fail=False,\n                            check_fake_dates=False,\n                            index_file=None,\n                            updates_tree=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git commands\n        for proof of concept without the need to figure out, how this is done\n        via GitPython.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str or list\n          arbitrary command str. `files` is appended to that string.\n        updates_tree: bool\n          whether or not command updates the working tree. If True, triggers\n          necessary reevaluations like self.config.reload()\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n\n        # ensure cmd_str becomes a well-formed list:\n        if isinstance(cmd_str, str):\n            cmd = split_cmdline(cmd_str)\n        else:\n            cmd = cmd_str[:]  # we will modify in-place\n\n        assert(cmd[0] == 'git')\n        cmd = cmd[:1] + self._GIT_COMMON_OPTIONS + cmd[1:]\n\n        if check_fake_dates and self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if index_file:\n            env = (env if env is not None else os.environ).copy()\n            env['GIT_INDEX_FILE'] = index_file\n\n        # TODO?: wouldn't splitting interfer with above GIT_INDEX_FILE\n        #  handling????\n        try:\n            out, err = run_gitcommand_on_file_list_chunks(\n                self.cmd_call_wrapper.run,\n                cmd,\n                files,\n                log_stderr=log_stderr,\n                log_stdout=log_stdout,\n                log_online=log_online,\n                expect_stderr=expect_stderr,\n                cwd=cwd,\n                env=env,\n                shell=shell,\n                expect_fail=expect_fail)\n        except CommandError as e:\n            ignored = re.search(GitIgnoreError.pattern, e.stderr)\n            if ignored:\n                raise GitIgnoreError(cmd=e.cmd, msg=e.stderr,\n                                     code=e.code, stdout=e.stdout,\n                                     stderr=e.stderr,\n                                     paths=ignored.groups()[0].splitlines())\n            raise\n\n        if updates_tree:\n            lgr.debug(\"Reloading config due to supposed working tree update\")\n            self.config.reload()\n\n        return out, err\n\n    # Convenience wrappers for one-off git calls that don't require further\n    # processing or error handling.\n\n    def call_git(self, args, files=None,\n                 expect_stderr=False, expect_fail=False):\n        \"\"\"Call git and return standard output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        expect_fail : bool, optional\n          A non-zero exit is expected and should not be elevated above the\n          DEBUG level.\n\n        Returns\n        -------\n        standard output (str)\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          expect_fail=expect_fail,\n                                          check_fake_dates=True)\n        return out\n\n    def call_git_items_(self, args, files=None, expect_stderr=False, sep=None):\n        \"\"\"Call git, splitting output on `sep`.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Returns\n        -------\n        Generator that yields output items.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          check_fake_dates=True)\n        yield from (out.split(sep) if sep else out.splitlines())\n\n    def call_git_oneline(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git for a single line of output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        AssertionError if there is more than one line of output.\n        \"\"\"\n        lines = list(self.call_git_items_(args, files=files,\n                                          expect_stderr=expect_stderr))\n        if len(lines) > 1:\n            raise AssertionError(\n                \"Expected {} to return single line, but it returned {}\"\n                .format([\"git\"] + args, lines))\n        return lines[0]\n\n    def call_git_success(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git and return true if the call exit code of 0.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        try:\n            self._git_custom_command(files, [\"git\"] + args,\n                                     expect_fail=True,\n                                     expect_stderr=expect_stderr,\n                                     check_fake_dates=True)\n        except CommandError:\n            return False\n        return True\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Register remote pointing to a url\n        \"\"\"\n        cmd = ['git', 'remote', 'add']\n        if options:\n            cmd += options\n        cmd += [name, url]\n\n        result = self._git_custom_command('', cmd)\n        self.config.reload()\n        return result\n\n    def remove_remote(self, name):\n        \"\"\"Remove existing remote\n        \"\"\"\n\n        # TODO: testing and error handling!\n        from .exceptions import RemoteNotAvailableError\n        try:\n            out, err = self._git_custom_command(\n                '', ['git', 'remote', 'remove', name])\n        except CommandError as e:\n            if 'fatal: No such remote' in e.stderr:\n                raise RemoteNotAvailableError(name,\n                                              cmd=\"git remote remove\",\n                                              msg=\"No such remote\",\n                                              stdout=e.stdout,\n                                              stderr=e.stderr)\n            else:\n                raise e\n\n        # TODO: config.reload necessary?\n        self.config.reload()\n        return\n\n    def update_remote(self, name=None, verbose=False):\n        \"\"\"\n        \"\"\"\n        options = [\"-v\"] if verbose else []\n        name = [name] if name else []\n        self._git_custom_command(\n            '', ['git', 'remote'] + name + ['update'] + options,\n            expect_stderr=True\n        )\n\n    def fetch(self, remote=None, refspec=None, all_=False, git_options=None,\n              **kwargs):\n        \"\"\"Fetches changes from a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to fetch from. If no remote is given and\n          `all_` is not set, the tracking branch is fetched.\n        refspec : str, optional\n          refspec to fetch.\n        all_ : bool, optional\n          fetch all remotes (and all of their branches).\n          Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-fetch.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-fetch.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        return list(\n            self.fetch_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    @guard_BadName\n    def fetch_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `fetch`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'fetch', '--verbose', '--progress'],\n            action='fetch',\n            urlvars=('remote.{}.url', 'remote.{}.url'),\n            protocol=GitProgress,\n            info_cls=FetchInfo,\n            info_from=1,\n            add_remote=False,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    # XXX Consider removing this method. It is only used in `update()`,\n    # where it could be easily replaced with fetch+merge\n    def pull(self, remote=None, refspec=None, git_options=None, **kwargs):\n        \"\"\"Pulls changes from a remote.\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to pull from. If no remote is given,\n          the remote tracking branch is used.\n        refspec : str, optional\n          refspec to fetch.\n        git_options : list, optional\n          Additional command line options for git-pull.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-pull.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n\n        cmd = ['git', 'pull', '--progress'] + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or fetch all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            # No explicit remote to fetch.\n            # => get tracking branch:\n            tb_remote, refspec = self.get_tracking_branch()\n            if tb_remote is not None:\n                remote = tb_remote\n            else:\n                # No remote, no tracking branch\n                # => fail\n                raise ValueError(\"Neither a remote is specified to pull \"\n                                 \"from nor a tracking branch is set up.\")\n\n        cmd.append(remote)\n        if refspec:\n            cmd += ensure_list(refspec)\n\n        # best effort to enable SSH connection caching\n        url = self.config.get('remote.{}.url'.format(remote), None)\n        if url and is_ssh(url):\n            ssh_manager.get_connection(url).open()\n        WitlessRunner(\n            cwd=self.path,\n            env=GitRunner.get_git_environ_adjusted()).run(\n                cmd,\n                protocol=StdOutCaptureWithGitProgress,\n        )\n\n    def push(self, remote=None, refspec=None, all_remotes=False,\n             all_=False, git_options=None, **kwargs):\n        \"\"\"Push changes to a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to push to. If no remote is given and\n          `all_` is not set, the tracking branch is pushed.\n        refspec : str, optional\n          refspec to push.\n        all_ : bool, optional\n          push to all remotes. Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-push.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-push.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        if all_remotes:\n            # be nice to the elderly\n            all_ = True\n        return list(\n            self.push_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    def push_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `push`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'push', '--progress', '--porcelain'],\n            action='push',\n            urlvars=('remote.{}.pushurl', 'remote.{}.url'),\n            protocol=StdOutCaptureWithGitProgress,\n            info_cls=PushInfo,\n            info_from=0,\n            add_remote=True,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    def _fetch_push_helper(\n            self,\n            base_cmd,     # arg list\n            action,       # label fetch|push\n            urlvars,      # variables to query for URLs\n            protocol,     # processor for output\n            info_cls,     # Push|FetchInfo\n            info_from,    # 0=stdout, 1=stderr\n            add_remote,   # whether to add a 'remote' field to the info dict\n            remote=None, refspec=None, all_=False, git_options=None):\n\n        git_options = ensure_list(git_options)\n\n        cmd = base_cmd + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or push all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            if all_:\n                remotes_to_process = self.get_remotes(with_urls_only=True)\n            else:\n                # No explicit remote to fetch.\n                # => get tracking branch:\n                tb_remote, refspec = self.get_tracking_branch()\n                if tb_remote is not None:\n                    remotes_to_process = [tb_remote]\n                else:\n                    # No remote, no tracking branch\n                    # => fail\n                    raise ValueError(\n                        \"Neither a remote is specified to {} \"\n                        \"from nor a tracking branch is set up.\".format(action))\n        else:\n            if all_:\n                raise ValueError(\n                    \"Option 'all_' conflicts with specified remote \"\n                    \"'{}'.\".format(remote))\n            remotes_to_process = [remote]\n\n        if refspec:\n            # prep for appending to cmd\n            refspec = ensure_list(refspec)\n\n        # no need for progress report, when there is just one remote\n        log_remote_progress = len(remotes_to_process) > 1\n        if log_remote_progress:\n            pbar_id = '{}remotes-{}'.format(action, id(self))\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start %sing remotes for %s', action, self,\n                total=len(remotes_to_process),\n                label=action.capitalize(),\n                unit=' Remotes',\n            )\n        try:\n            for remote in remotes_to_process:\n                r_cmd = cmd + [remote]\n                if refspec:\n                    r_cmd += refspec\n\n                if log_remote_progress:\n                    log_progress(\n                        lgr.info,\n                        pbar_id,\n                        '{}ing remote %s'.format(action.capitalize()),\n                        remote,\n                        update=1,\n                        increment=True,\n                    )\n                # best effort to enable SSH connection caching\n                url = self.config.get(\n                    # make two attempts to get a URL\n                    urlvars[0].format(remote),\n                    self.config.get(\n                        urlvars[1].format(remote),\n                        None)\n                )\n                if url and is_ssh(url):\n                    ssh_manager.get_connection(url).open()\n                try:\n                    out = WitlessRunner(\n                        cwd=self.path,\n                        env=GitRunner.get_git_environ_adjusted()).run(\n                            r_cmd,\n                            protocol=protocol,\n                    )\n                    output = out[info_from] or ''\n                except CommandError as e:\n                    # intercept some errors that we express as an error report\n                    # in the info dicts\n                    if re.match('error: failed to (push|fetch) some refs', e.stderr):\n                        output = {1: e.stderr, 0: e.stdout}[info_from]\n                        if output is None:\n                            output = ''\n                    else:\n                        raise\n\n                for line in output.splitlines():\n                    try:\n                        # push info doesn't identify a remote, add it here\n                        pi = info_cls._from_line(line)\n                        if add_remote:\n                            pi['remote'] = remote\n                        yield pi\n                    except Exception:\n                        # it is not progress and no push info\n                        # don't hide it completely\n                        lgr.debug('git-%s reported: %s', action, line)\n        finally:\n            if log_remote_progress:\n                log_progress(\n                    lgr.info,\n                    pbar_id,\n                    'Finished %sing remotes for %s', action, self,\n                )\n\n    def get_remote_url(self, name, push=False):\n        \"\"\"Get the url of a remote.\n\n        Reads the configuration of remote `name` and returns its url or None,\n        if there is no url configured.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        push: bool\n          if True, get the pushurl instead of the fetch url.\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        return self.config.get(var, None)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        self.config.set(var, url, where='local', reload=True)\n\n    def get_branch_commits_(self, branch=None, limit=None, stop=None):\n        \"\"\"Return commit hexshas for a branch\n\n        Parameters\n        ----------\n        branch: str, optional\n          If not provided, assumes current branch\n        limit: None | 'left-only', optional\n          Limit which commits to report.  If None -- all commits (merged or not),\n          if 'left-only' -- only the commits from the left side of the tree upon\n          merges\n        stop: str, optional\n          hexsha of the commit at which stop reporting (matched one is not\n          reported either)\n\n        Yields\n        ------\n        str\n        \"\"\"\n        cmd = ['rev-list']\n        if limit == 'left-only':\n            cmd.append('--left-only')\n        if not branch:\n            branch = self.get_active_branch()\n        cmd.append(branch)\n        for r in self.call_git_items_(cmd):\n            if stop and stop == r:\n                return\n            yield r\n\n    def checkout(self, name, options=None):\n        \"\"\"\n        \"\"\"\n        # TODO: May be check for the need of -b options herein?\n        cmd = ['git', 'checkout']\n        if options:\n            cmd += options\n        cmd += [str(name)]\n\n        self._git_custom_command('', cmd, expect_stderr=True, updates_tree=True)\n\n    # TODO: Before implementing annex merge, find usages and check for a needed\n    # change to call super().merge\n    def merge(self, name, options=None, msg=None, allow_unrelated=False, **kwargs):\n        if options is None:\n            options = []\n        if msg:\n            options = options + [\"-m\", msg]\n        if allow_unrelated and external_versions['cmd:git'] >= '2.9':\n            options += ['--allow-unrelated-histories']\n        self._git_custom_command(\n            '', ['git', 'merge'] + options + [name],\n            check_fake_dates=True,\n            **kwargs\n        )\n\n    def remove_branch(self, branch):\n        self._git_custom_command(\n            '', ['git', 'branch', '-D', branch]\n        )\n\n    def cherry_pick(self, commit):\n        \"\"\"Cherry pick `commit` to the current branch.\n\n        Parameters\n        ----------\n        commit : str\n            A single commit.\n        \"\"\"\n        self._git_custom_command(\"\", [\"git\", \"cherry-pick\", commit],\n                                 check_fake_dates=True)\n\n    @property\n    def dirty(self):\n        \"\"\"Is the repository dirty?\n\n        Note: This provides a quick answer when you simply want to know if\n        there are any untracked changes or modifications in this repository or\n        its submodules. For finer-grained control and more detailed reporting,\n        use status() instead.\n        \"\"\"\n        stdout, _ = self._git_custom_command(\n            [],\n            [\"git\", \"status\", \"--porcelain\",\n             # Ensure the result isn't influenced by status.showUntrackedFiles.\n             \"--untracked-files=normal\",\n             # Ensure the result isn't influenced by diff.ignoreSubmodules.\n             \"--ignore-submodules=none\"])\n        return bool(stdout.strip())\n\n    @property\n    def untracked_files(self):\n        \"\"\"Legacy interface, do not use! Use the status() method instead.\n\n        Despite its name, it also reports on untracked datasets, and\n        yields their names with trailing path separators.\n        \"\"\"\n        return [\n            '{}{}'.format(\n                str(p.relative_to(self.pathobj)),\n                os.sep if props['type'] != 'file' else ''\n            )\n            for p, props in self.status(\n                    untracked='all', eval_submodule_state='no').items()\n            if props.get('state', None) == 'untracked'\n        ]\n\n    def gc(self, allow_background=False, auto=False):\n        \"\"\"Perform house keeping (garbage collection, repacking)\"\"\"\n        cmd_options = ['git']\n        if not allow_background:\n            cmd_options += ['-c', 'gc.autodetach=0']\n        cmd_options += ['gc', '--aggressive']\n        if auto:\n            cmd_options += ['--auto']\n        self._git_custom_command('', cmd_options)\n\n    def _parse_gitmodules(self):\n        # TODO read .gitconfig from Git blob?\n        gitmodules = self.pathobj / '.gitmodules'\n        if not gitmodules.exists():\n            return {}\n        # pull out file content\n        out, err = self._git_custom_command(\n            '',\n            ['git', 'config', '-z', '-l', '--file', '.gitmodules'])\n        # abuse our config parser\n        db, _ = _parse_gitconfig_dump(out, {}, None, True, cwd=self.path)\n        mods = {}\n        for k, v in db.items():\n            if not k.startswith('submodule.'):\n                # we don't know what this is\n                lgr.warning(\"Skip unrecognized .gitmodule specification: %s=%s\", k, v)\n                continue\n            k_l = k.split('.')\n            # module name is everything after 'submodule.' that is not the variable\n            # name\n            mod_name = '.'.join(k_l[1:-1])\n            mod = mods.get(mod_name, {})\n            # variable name is the last 'dot-free' segment in the key\n            mod[k_l[-1]] = v\n            mods[mod_name] = mod\n\n        out = {}\n        # bring into traditional shape\n        for name, props in mods.items():\n            if 'path' not in props:\n                lgr.warning(\"Failed to get '%s.path', skipping this submodule\", name)\n                continue\n            modprops = {'gitmodule_{}'.format(k): v\n                        for k, v in props.items()\n                        if not (k.startswith('__') or k == 'path')}\n            modpath = self.pathobj / PurePosixPath(props['path'])\n            modprops['gitmodule_name'] = name\n            out[modpath] = modprops\n        return out\n\n    def get_submodules_(self, paths=None):\n        \"\"\"Yield submodules in this repository.\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n\n        Returns\n        -------\n        A generator that yields a dictionary with information for each\n        submodule.\n        \"\"\"\n        if not (self.pathobj / \".gitmodules\").exists():\n            return\n\n        modinfo = self._parse_gitmodules()\n        for path, props in self.get_content_info(\n                paths=paths,\n                ref=None,\n                untracked='no',\n                eval_file_type=False).items():\n            if props.get('type', None) != 'dataset':\n                continue\n            props[\"path\"] = path\n            props.update(modinfo.get(path, {}))\n            yield props\n\n    def get_submodules(self, sorted_=True, paths=None, compat=True):\n        \"\"\"Return list of submodules.\n\n        Parameters\n        ----------\n        sorted_ : bool, optional\n            Sort submodules by path name.\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n        compat : bool, optional\n            If true, return a namedtuple that incompletely mimics the\n            attributes of GitPython's Submodule object in hope of backwards\n            compatibility with previous callers. Note that this form should be\n            considered temporary and callers should be updated; this flag will\n            be removed in a future release.\n\n        Returns\n        -------\n        List of submodule namedtuples if `compat` is true or otherwise a list\n        of dictionaries as returned by `get_submodules_`.\n        \"\"\"\n        xs = self.get_submodules_(paths=paths)\n        if compat:\n            warnings.warn(\"The attribute-based return value of get_submodules() \"\n                          \"exists for compatibility purposes and will be removed \"\n                          \"in an upcoming release\",\n                          DeprecationWarning)\n            xs = (Submodule(name=p[\"gitmodule_name\"],\n                            path=str(p[\"path\"].relative_to(self.pathobj)),\n                            url=p[\"gitmodule_url\"])\n                  for p in xs)\n\n        if sorted_:\n            if compat:\n                def key(x):\n                    return x.path\n            else:\n                def key(x):\n                    return x[\"path\"]\n            xs = sorted(xs, key=key)\n        return list(xs)\n\n    def add_submodule(self, path, name=None, url=None, branch=None):\n        \"\"\"Add a new submodule to the repository.\n\n        This will alter the index as well as the .gitmodules file, but will not\n        create a new commit.  If the submodule already exists, no matter if the\n        configuration differs from the one provided, the existing submodule\n        is considered as already added and no further action is performed.\n\n        Parameters\n        ----------\n        path : str\n          repository-relative path at which the submodule should be located, and\n          which will be created as required during the repository initialization.\n        name : str or None\n          name/identifier for the submodule. If `None`, the `path` will be used\n          as name.\n        url : str or None\n          git-clone compatible URL. If `None`, the repository is assumed to\n          exist, and the url of the first remote is taken instead. This is\n          useful if you want to make an existing repository a submodule of\n          another one.\n        branch : str or None\n          name of branch to be checked out in the submodule. The given branch\n          must exist in the remote repository, and will be checked out locally\n          as a tracking branch. If `None`, remote HEAD will be checked out.\n        \"\"\"\n        if name is None:\n            name = Path(path).as_posix()\n        # XXX the following should do it, but GitPython will refuse to add a submodule\n        # unless you specify a URL that is configured as one of its remotes, or you\n        # specify no URL, but the repo has at least one remote.\n        # this is stupid, as for us it is valid to not have any remote, because we can\n        # still obtain the submodule from a future publication location, based on the\n        # parent\n        # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch)\n        # going git native instead\n        cmd = ['submodule', 'add', '--name', name]\n        if branch is not None:\n            cmd += ['-b', branch]\n        if url is None:\n            # repo must already exist locally\n            subm = GitRepo(op.join(self.path, path), create=False, init=False)\n            # check that it has a commit, and refuse\n            # to operate on it otherwise, or we would get a bastard\n            # submodule that cripples git operations\n            if not subm.get_hexsha():\n                raise InvalidGitRepositoryError(\n                    'cannot add subdataset {} with no commits'.format(subm))\n            # make an attempt to configure a submodule source URL based on the\n            # discovered remote configuration\n            remote, branch = subm.get_tracking_branch()\n            url = subm.get_remote_url(remote) if remote else None\n\n        if url is None:\n            # had no luck with a remote URL\n            if not isabs(path):\n                # need to recode into a relative path \"URL\" in POSIX\n                # style, even on windows\n                url = posixpath.join(curdir, posix_relpath(path))\n            else:\n                url = path\n        cmd += [url, Path(path).as_posix()]\n        self.call_git(cmd)\n        # record dataset ID if possible for comprehesive metadata on\n        # dataset components within the dataset itself\n        subm_id = GitRepo(op.join(self.path, path)).config.get(\n            'datalad.dataset.id', None)\n        if subm_id:\n            self.call_git(\n                ['config', '--file', '.gitmodules', '--replace-all',\n                 'submodule.{}.datalad-id'.format(name), subm_id])\n        # ensure supported setup\n        _fixup_submodule_dotgit_setup(self, path)\n        # TODO: return value\n\n    def deinit_submodule(self, path, **kwargs):\n        \"\"\"Deinit a submodule\n\n        Parameters\n        ----------\n        path: str\n            path to the submodule; relative to `self.path`\n        kwargs:\n            see `__init__`\n        \"\"\"\n\n        self._git_custom_command(path,\n                                 ['git', 'submodule', 'deinit'] +\n                                 to_options(**kwargs))\n        # TODO: return value\n\n    def update_submodule(self, path, mode='checkout', init=False):\n        \"\"\"Update a registered submodule.\n\n        This will make the submodule match what the superproject expects by\n        cloning missing submodules and updating the working tree of the\n        submodules. The \"updating\" can be done in several ways depending\n        on the value of submodule.<name>.update configuration variable, or\n        the `mode` argument.\n\n        Parameters\n        ----------\n        path : str\n          Identifies which submodule to operate on by it's repository-relative\n          path.\n        mode : {checkout, rebase, merge}\n          Update procedure to perform. 'checkout': the commit recorded in the\n          superproject will be checked out in the submodule on a detached HEAD;\n          'rebase': the current branch of the submodule will be rebased onto\n          the commit recorded in the superproject; 'merge': the commit recorded\n          in the superproject will be merged into the current branch in the\n          submodule.\n        init : bool\n          If True, initialize all submodules for which \"git submodule init\" has\n          not been called so far before updating.\n          Primarily provided for internal purposes and should not be used directly\n          since would result in not so annex-friendly .git symlinks/references\n          instead of full featured .git/ directories in the submodules\n        \"\"\"\n        if GitRepo.is_valid_repo(self.pathobj / path):\n            subrepo = GitRepo(self.pathobj / path, create=False)\n            subbranch = subrepo.get_active_branch() if subrepo else None\n            subbranch_hexsha = subrepo.get_hexsha(subbranch) if subrepo else None\n        else:\n            subrepo = None\n            subbranch = None\n            subbranch_hexsha = None\n\n        cmd = ['git', 'submodule', 'update', '--%s' % mode]\n        if init:\n            cmd.append('--init')\n            subgitpath = opj(self.path, path, '.git')\n            if not exists(subgitpath):\n                # TODO:  wouldn't with --init we get all those symlink'ed .git/?\n                # At least let's warn\n                lgr.warning(\n                    \"Do not use update_submodule with init=True to avoid git creating \"\n                    \"symlinked .git/ directories in submodules\"\n                )\n            #  yoh: I thought I saw one recently but thought it was some kind of\n            #  an artifact from running submodule update --init manually at\n            #  some point, but looking at this code now I worry that it was not\n        self._git_custom_command(path, cmd)\n\n        if not init:\n            return\n\n        # track branch originally cloned, only if we had a valid repo at the start\n        updated_subbranch = subrepo.get_active_branch() if subrepo else None\n        if subbranch and not updated_subbranch:\n            # got into 'detached' mode\n            # trace if current state is a predecessor of the branch_hexsha\n            lgr.debug(\n                \"Detected detached HEAD after updating submodule %s which was \"\n                \"in %s branch before\", self.path, subbranch)\n            detached_hexsha = subrepo.get_hexsha()\n            if subrepo.get_merge_base(\n                    [subbranch_hexsha, detached_hexsha]) == detached_hexsha:\n                # TODO: config option?\n                # in all likely event it is of the same branch since\n                # it is an ancestor -- so we could update that original branch\n                # to point to the state desired by the submodule, and update\n                # HEAD to point to that location\n                lgr.info(\n                    \"Submodule HEAD got detached. Resetting branch %s to point \"\n                    \"to %s. Original location was %s\",\n                    subbranch, detached_hexsha[:8], subbranch_hexsha[:8]\n                )\n                branch_ref = 'refs/heads/%s' % subbranch\n                subrepo.update_ref(branch_ref, detached_hexsha)\n                assert(subrepo.get_hexsha(subbranch) == detached_hexsha)\n                subrepo.update_ref('HEAD', branch_ref, symbolic=True)\n                assert(subrepo.get_active_branch() == subbranch)\n            else:\n                lgr.warning(\n                    \"%s has a detached HEAD since cloned branch %s has another common ancestor with %s\",\n                    subrepo.path, subbranch, detached_hexsha[:8]\n                )\n        # TODO: return value\n\n    def update_ref(self, ref, value, symbolic=False):\n        \"\"\"Update the object name stored in a ref \"safely\".\n\n        Just a shim for `git update-ref` call if not symbolic, and\n        `git symbolic-ref` if symbolic\n\n        Parameters\n        ----------\n        ref : str\n          Reference, such as `ref/heads/BRANCHNAME` or HEAD.\n        value : str\n          Value to update to, e.g. hexsha of a commit when updating for a\n          branch ref, or branch ref if updating HEAD\n        symbolic : None\n          To instruct if ref is symbolic, e.g. should be used in case of\n          ref=HEAD\n        \"\"\"\n        self._git_custom_command(\n            '',\n            ['git', 'symbolic-ref' if symbolic else 'update-ref', ref, value]\n        )\n\n    def tag(self, tag, message=None, commit=None, options=None):\n        \"\"\"Tag a commit\n\n        Parameters\n        ----------\n        tag : str\n          Custom tag label. Must be a valid tag name.\n        message : str, optional\n          If provided, adds ['-m', <message>] to the list of `git tag`\n          arguments.\n        commit : str, optional\n          If provided, will be appended as last argument to the `git tag` call,\n          and can be used to identify the commit that shall be tagged, if\n          not HEAD.\n        options : list, optional\n          Additional command options, inserted prior a potential `commit`\n          argument.\n        \"\"\"\n        # TODO: call in save.py complains about extensive logging. When does it\n        # happen in what way? Figure out, whether to just silence it or raise or\n        # whatever else.\n        args = ['tag']\n        if message:\n            args += ['-m', message]\n        if options is not None:\n            args.extend(options)\n        args.append(tag)\n        if commit:\n            args.append(commit)\n        self.call_git(args)\n\n    def get_tags(self, output=None):\n        \"\"\"Get list of tags\n\n        Parameters\n        ----------\n        output : str, optional\n          If given, limit the return value to a list of values matching that\n          particular key of the tag properties.\n\n        Returns\n        -------\n        list\n          Each item is a dictionary with information on a tag. At present\n          this includes 'hexsha', and 'name', where the latter is the string\n          label of the tag, and the former the hexsha of the object the tag\n          is attached to. The list is sorted by the creator date (committer\n          date for lightweight tags and tagger date for annotated tags), with\n          the most recent commit being the last element.\n        \"\"\"\n        tags = [\n            dict(\n                name=t['refname:strip=2'],\n                hexsha=t['object'] if t['object'] else t['objectname'],\n            )\n            for t in self.for_each_ref_(\n                fields=['refname:strip=2', 'objectname', 'object'],\n                pattern='refs/tags',\n                sort='creatordate')\n        ]\n        if output:\n            return [t[output] for t in tags]\n        else:\n            return tags\n\n    def describe(self, commitish=None, **kwargs):\n        \"\"\" Quick and dirty implementation to call git-describe\n\n        Parameters\n        ----------\n        kwargs:\n            transformed to cmdline options for git-describe;\n            see __init__ for description of the transformation\n        \"\"\"\n        # TODO: be more precise what failure to expect when and raise actual\n        # errors\n        cmd = ['git', 'describe'] + to_options(**kwargs)\n        if commitish is not None:\n            cmd.append(commitish)\n        try:\n            describe, outerr = self._git_custom_command(\n                [],\n                cmd,\n                expect_fail=True)\n            return describe.strip()\n        # TODO: WTF \"catch everything\"?\n        except:\n            return None\n\n    def get_tracking_branch(self, branch=None, remote_only=False):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        Parameters\n        ----------\n        branch: str\n            local branch to look up. If none is given, active branch is used.\n        remote_only : bool\n            Don't return a value if the upstream remote is set to \".\" (meaning\n            this repository).\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n        if branch is None:\n            branch = self.get_active_branch()\n            if branch is None:\n                return None, None\n\n        track_remote = self.config.get('branch.{0}.remote'.format(branch), None)\n        if remote_only and track_remote == \".\":\n            return None, None\n        track_branch = self.config.get('branch.{0}.merge'.format(branch), None)\n        return track_remote, track_branch\n\n    @property\n    def count_objects(self):\n        \"\"\"return dictionary with count, size(in KiB) information of git objects\n        \"\"\"\n\n        count_cmd = ['git', 'count-objects', '-v']\n        count_str, err = self._git_custom_command('', count_cmd)\n        count = {key: int(value)\n                 for key, value in [item.split(': ')\n                                    for item in count_str.split('\\n')\n                                    if len(item.split(': ')) == 2]}\n        return count\n\n    def get_git_attributes(self):\n        \"\"\"Query gitattributes which apply to top level directory\n\n        It is a thin compatibility/shortcut wrapper around more versatile\n        get_gitattributes which operates on a list of paths and returns\n        a dictionary per each path\n\n        Returns\n        -------\n        dict:\n          a dictionary with attribute name and value items relevant for the\n          top ('.') directory of the repository, and thus most likely the\n          default ones (if not overwritten with more rules) for all files within\n          repo.\n        \"\"\"\n        return self.get_gitattributes('.')['.']\n\n    def get_gitattributes(self, path, index_only=False):\n        \"\"\"Query gitattributes for one or more paths\n\n        Parameters\n        ----------\n        path: path or list\n          Path(s) to query. Paths may be relative or absolute.\n        index_only: bool\n          Flag whether to consider only gitattribute setting that are reflected\n          in the repository index, not just in the work tree content.\n\n        Returns\n        -------\n        dict:\n          Each key is a queried path (always relative to the repostiory root),\n          each value is a dictionary with attribute\n          name and value items. Attribute values are either True or False,\n          for set and unset attributes, or are the literal attribute value.\n        \"\"\"\n        path = ensure_list(path)\n        cmd = [\"git\", \"check-attr\", \"-z\", \"--all\"]\n        if index_only:\n            cmd.append('--cached')\n        stdout, stderr = self._git_custom_command(path, cmd)\n        # make sure we have one entry for each query path to\n        # simplify work with the result\n        attributes = {_normalize_path(self.path, p): {} for p in path}\n        attr = []\n        for item in stdout.split('\\0'):\n            attr.append(item)\n            if len(attr) < 3:\n                continue\n            # we have a full record\n            p, name, value = attr\n            attrs = attributes[p]\n            attrs[name] = \\\n                True if value == 'set' else False if value == 'unset' else value\n            # done, reset item\n            attr = []\n        return attributes\n\n    def set_gitattributes(self, attrs, attrfile='.gitattributes', mode='a'):\n        \"\"\"Set gitattributes\n\n        By default appends additional lines to `attrfile`. Note, that later\n        lines in `attrfile` overrule earlier ones, which may or may not be\n        what you want. Set `mode` to 'w' to replace the entire file by\n        what you provided in `attrs`.\n\n        Parameters\n        ----------\n        attrs : list\n          Each item is a 2-tuple, where the first element is a path pattern,\n          and the second element is a dictionary with attribute key/value\n          pairs. The attribute dictionary must use the same semantics as those\n          returned by `get_gitattributes()`. Path patterns can use absolute paths,\n          in which case they will be normalized relative to the directory\n          that contains the target .gitattributes file (see `attrfile`).\n        attrfile: path\n          Path relative to the repository root of the .gitattributes file the\n          attributes shall be set in.\n        mode: str\n          'a' to append .gitattributes, 'w' to replace it\n        \"\"\"\n\n        git_attributes_file = op.join(self.path, attrfile)\n        attrdir = op.dirname(git_attributes_file)\n        if not op.exists(attrdir):\n            os.makedirs(attrdir)\n        with open(git_attributes_file, mode) as f:\n            for pattern, attr in sorted(attrs, key=lambda x: x[0]):\n                # normalize the pattern relative to the target .gitattributes file\n                npath = _normalize_path(\n                    op.join(self.path, op.dirname(attrfile)), pattern)\n                # paths in gitattributes always have to be POSIX\n                npath = Path(npath).as_posix()\n                attrline = u''\n                if npath.count(' '):\n                    # quote patterns with spaces\n                    attrline += u'\"{}\"'.format(npath.replace('\"', '\\\\\"'))\n                else:\n                    attrline += npath\n                for a in sorted(attr):\n                    val = attr[a]\n                    if val is True:\n                        attrline += ' {}'.format(a)\n                    elif val is False:\n                        attrline += ' -{}'.format(a)\n                    else:\n                        attrline += ' {}={}'.format(a, val)\n                f.write('\\n{}'.format(attrline))\n\n    def get_content_info(self, paths=None, ref=None, untracked='all',\n                         eval_file_type=True):\n        \"\"\"Get identifier and type information from repository content.\n\n        This is simplified front-end for `git ls-files/tree`.\n\n        Both commands differ in their behavior when queried about subdataset\n        paths. ls-files will not report anything, ls-tree will report on the\n        subdataset record. This function uniformly follows the behavior of\n        ls-tree (report on the respective subdataset mount).\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath)\n          Specific paths, relative to the resolved repository root, to query\n          info for. Paths must be normed to match the reporting done by Git,\n          i.e. no parent dir components (ala \"some/../this\").\n          If none are given, info is reported for all content.\n        ref : gitref or None\n          If given, content information is retrieved for this Git reference\n          (via ls-tree), otherwise content information is produced for the\n          present work tree (via ls-files). With a given reference, the\n          reported content properties also contain a 'bytesize' record,\n          stating the size of a file in bytes.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when no `ref` was given:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_file_type : bool\n          If True, inspect file type of untracked files, and report annex\n          symlink pointers as type 'file'. This convenience comes with a\n          cost; disable to get faster performance if this information\n          is not needed.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n\n            Note that the reported type will not always match the type of\n            content committed to Git, rather it will reflect the nature\n            of the content minus platform/mode-specifics. For example,\n            a symlink to a locked annexed file on Unix will have a type\n            'file', reported, while a symlink to a file in Git or directory\n            will be of type 'symlink'.\n\n          `gitshasum`\n            SHASUM of the item as tracked by Git, or None, if not\n            tracked. This could be different from the SHASUM of the file\n            in the worktree, if it was modified.\n\n        Raises\n        ------\n        ValueError\n          In case of an invalid Git reference (e.g. 'HEAD' in an empty\n          repository)\n        \"\"\"\n        lgr.debug('%s.get_content_info(...)', self)\n        # TODO limit by file type to replace code in subdatasets command\n        info = OrderedDict()\n\n        if paths:\n            # path matching will happen against what Git reports\n            # and Git always reports POSIX paths\n            # any incoming path has to be relative already, so we can simply\n            # convert unconditionally\n            paths = [ut.PurePosixPath(p) for p in paths]\n\n        path_strs = list(map(str, paths)) if paths else None\n\n        # this will not work in direct mode, but everything else should be\n        # just fine\n        if not ref:\n            # make sure no operations are pending before we figure things\n            # out in the worktree\n            self.precommit()\n\n            # --exclude-standard will make sure to honor and standard way\n            # git can be instructed to ignore content, and will prevent\n            # crap from contaminating untracked file reports\n            cmd = ['git', 'ls-files',\n                   '--stage', '-z', '-d', '-m', '--exclude-standard']\n            # untracked report mode, using labels from `git diff` option style\n            if untracked == 'all':\n                cmd.append('-o')\n            elif untracked == 'normal':\n                cmd += ['-o', '--directory', '--no-empty-directory']\n            elif untracked == 'no':\n                pass\n            else:\n                raise ValueError(\n                    'unknown value for `untracked`: {}'.format(untracked))\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) (?P<sha>.*) (.*)\\t(?P<fname>.*)$')\n\n            if path_strs:\n                # we need to get their within repo elements since ls-tree\n                # for paths within submodules returns nothing!\n                # see https://public-inbox.org/git/20190703193305.GF21553@hopa.kiewit.dartmouth.edu/T/#u\n                submodules = [s.path for s in self.get_submodules()]\n                path_strs = get_parent_paths(path_strs, submodules)\n        else:\n            cmd = ['git', 'ls-tree', ref, '-z', '-r', '--full-tree', '-l']\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) ([a-z]*) (?P<sha>[^ ]*) [\\s]*(?P<size>[0-9-]+)\\t(?P<fname>.*)$')\n\n        lgr.debug('Query repo: %s', cmd)\n        try:\n            stdout, stderr = self._git_custom_command(\n                path_strs,\n                cmd,\n                log_stderr=True,\n                log_stdout=True,\n                # not sure why exactly, but log_online has to be false!\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            if \"fatal: Not a valid object name\" in str(exc):\n                raise InvalidGitReferenceError(ref)\n            raise\n        lgr.debug('Done query repo: %s', cmd)\n\n        if not eval_file_type:\n            _get_link_target = None\n        elif ref:\n            def _read_symlink_target_from_catfile(lines):\n                # it is always the second line, all checks done upfront\n                header = lines.readline()\n                if header.rstrip().endswith('missing'):\n                    # something we do not know about, should not happen\n                    # in real use, but guard against to avoid stalling\n                    return ''\n                return lines.readline().rstrip()\n\n            _get_link_target = BatchedCommand(\n                ['git', 'cat-file', '--batch'],\n                path=self.path,\n                output_proc=_read_symlink_target_from_catfile,\n            )\n        else:\n            def try_readlink(path):\n                try:\n                    return os.readlink(path)\n                except OSError:\n                    # readlink will fail if the symlink reported by ls-files is\n                    # not in the working tree (it could be removed or\n                    # unlocked). Fall back to a slower method.\n                    return op.realpath(path)\n\n            _get_link_target = try_readlink\n\n        try:\n            self._get_content_info_line_helper(\n                ref,\n                info,\n                stdout.split('\\0'),\n                props_re,\n                _get_link_target)\n        finally:\n            if ref and _get_link_target:\n                # cancel batch process\n                _get_link_target.close()\n\n        lgr.debug('Done %s.get_content_info(...)', self)\n        return info\n\n    def _get_content_info_line_helper(self, ref, info, lines,\n                                      props_re, get_link_target):\n        \"\"\"Internal helper of get_content_info() to parse Git output\"\"\"\n        mode_type_map = {\n            '100644': 'file',\n            '100755': 'file',\n            '120000': 'symlink',\n            '160000': 'dataset',\n        }\n        for line in lines:\n            if not line:\n                continue\n            inf = {}\n            props = props_re.match(line)\n            if not props:\n                # not known to Git, but Git always reports POSIX\n                path = ut.PurePosixPath(line)\n                inf['gitshasum'] = None\n            else:\n                # again Git reports always in POSIX\n                path = ut.PurePosixPath(props.group('fname'))\n\n            # revisit the file props after this path has not been rejected\n            if props:\n                inf['gitshasum'] = props.group('sha')\n                inf['type'] = mode_type_map.get(\n                    props.group('type'), props.group('type'))\n                if get_link_target and inf['type'] == 'symlink' and \\\n                        ((ref is None and '.git/annex/objects' in \\\n                          ut.Path(\n                            get_link_target(str(self.pathobj / path))\n                          ).as_posix()) or \\\n                         (ref and \\\n                          '.git/annex/objects' in get_link_target(\n                              u'{}:{}'.format(\n                                  ref, str(path))))\n                        ):\n                    # report annex symlink pointers as file, their\n                    # symlink-nature is a technicality that is dependent\n                    # on the particular mode annex is in\n                    inf['type'] = 'file'\n\n                if ref and inf['type'] == 'file':\n                    inf['bytesize'] = int(props.group('size'))\n\n            # join item path with repo path to get a universally useful\n            # path representation with auto-conversion and tons of other\n            # stuff\n            path = self.pathobj.joinpath(path)\n            if 'type' not in inf:\n                # be nice and assign types for untracked content\n                inf['type'] = 'symlink' if path.is_symlink() \\\n                    else 'directory' if path.is_dir() else 'file'\n            info[path] = inf\n\n    def status(self, paths=None, untracked='all', eval_submodule_state='full'):\n        \"\"\"Simplified `git status` equivalent.\n\n        Parameters\n        ----------\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list. If a query path points\n          into a subdataset, a report is made on the subdataset record\n          within the queried dataset only (no recursion).\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        lgr.debug('Query status of %r for %s paths',\n                  self, len(paths) if paths else 'all')\n        return self.diffstatus(\n            fr='HEAD' if self.get_hexsha() else None,\n            to=None,\n            paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state)\n\n    def diff(self, fr, to, paths=None, untracked='all',\n             eval_submodule_state='full'):\n        \"\"\"Like status(), but reports changes between to arbitrary revisions\n\n        Parameters\n        ----------\n        fr : str or None\n          Revision specification (anything that Git understands). Passing\n          `None` considers anything in the target state as new.\n        to : str or None\n          Revision specification (anything that Git understands), or None\n          to compare to the state of the work tree.\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when `to` is None:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        return {k: v for k, v in self.diffstatus(\n            fr=fr, to=to, paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state).items()\n            if v.get('state', None) != 'clean'}\n\n    def diffstatus(self, fr, to, paths=None, untracked='all',\n                   eval_submodule_state='full', eval_file_type=True,\n                   _cache=None):\n        \"\"\"Like diff(), but reports the status of 'clean' content too\"\"\"\n        return self._diffstatus(\n            fr, to, paths, untracked, eval_submodule_state, eval_file_type,\n            _cache)\n\n    def _diffstatus(self, fr, to, paths, untracked, eval_state,\n                    eval_file_type, _cache):\n        \"\"\"Just like diffstatus(), but supports an additional evaluation\n        state 'global'. If given, it will return a single 'modified'\n        (vs. 'clean') state label for the entire repository, as soon as\n        it can.\"\"\"\n        def _get_cache_key(label, paths, ref, untracked=None):\n            return self.path, label, tuple(paths) if paths else None, \\\n                ref, untracked\n\n        if _cache is None:\n            _cache = {}\n\n        if paths:\n            # at this point we must normalize paths to the form that\n            # Git would report them, to easy matching later on\n            paths = [ut.Path(p) for p in paths]\n            paths = [\n                p.relative_to(self.pathobj) if p.is_absolute() else p\n                for p in paths\n            ]\n\n        # TODO report more info from get_content_info() calls in return\n        # value, those are cheap and possibly useful to a consumer\n        # we need (at most) three calls to git\n        if to is None:\n            # everything we know about the worktree, including os.stat\n            # for each file\n            key = _get_cache_key('ci', paths, None, untracked)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=None, untracked=untracked,\n                    eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we want Git to tell us what it considers modified and avoid\n            # reimplementing logic ourselves\n            key = _get_cache_key('mod', paths, None)\n            if key in _cache:\n                modified = _cache[key]\n            else:\n                modified = set(\n                    self.pathobj.joinpath(ut.PurePosixPath(p))\n                    for p in self._git_custom_command(\n                        # low-level code cannot handle pathobjs\n                        [str(p) for p in paths] if paths else None,\n                        ['git', 'ls-files', '-z', '-m'])[0].split('\\0')\n                    if p)\n                _cache[key] = modified\n        else:\n            key = _get_cache_key('ci', paths, to)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=to, eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we do not need worktree modification detection in this case\n            modified = None\n        # origin state\n        key = _get_cache_key('ci', paths, fr)\n        if key in _cache:\n            from_state = _cache[key]\n        else:\n            if fr:\n                from_state = self.get_content_info(\n                    paths=paths, ref=fr, eval_file_type=eval_file_type)\n            else:\n                # no ref means from nothing\n                from_state = {}\n            _cache[key] = from_state\n\n        status = OrderedDict()\n        for f, to_state_r in to_state.items():\n            props = None\n            if f not in from_state:\n                # this is new, or rather not known to the previous state\n                props = dict(\n                    state='added' if to_state_r['gitshasum'] else 'untracked',\n                )\n                if 'type' in to_state_r:\n                    props['type'] = to_state_r['type']\n            elif to_state_r['gitshasum'] == from_state[f]['gitshasum'] and \\\n                    (modified is None or f not in modified):\n                if to_state_r['type'] != 'dataset':\n                    # no change in git record, and no change on disk\n                    props = dict(\n                        # at this point we know that the reported object ids\n                        # for this file are identical in the to and from\n                        # records.  If to is None, we're comparing to the\n                        # working tree and a deleted file will still have an\n                        # identical id, so we need to check whether the file is\n                        # gone before declaring it clean. This working tree\n                        # check is irrelevant and wrong if to is a ref.\n                        state='clean' if to is not None or (f.exists() or \\\n                              f.is_symlink()) else 'deleted',\n                        type=to_state_r['type'],\n                    )\n                else:\n                    # a dataset\n                    props = dict(type=to_state_r['type'])\n                    if to is not None:\n                        # we can only be confident without looking\n                        # at the worktree, if we compare to a recorded\n                        # state\n                        props['state'] = 'clean'\n                    else:\n                        # report the shasum that we know, for further\n                        # wrangling of subdatasets below\n                        props['gitshasum'] = to_state_r['gitshasum']\n                        props['prev_gitshasum'] = from_state[f]['gitshasum']\n            else:\n                # change in git record, or on disk\n                props = dict(\n                    # TODO we could have a new file that is already staged\n                    # but had subsequent modifications done to it that are\n                    # unstaged. Such file would presently show up as 'added'\n                    # ATM I think this is OK, but worth stating...\n                    state='modified' if f.exists() or \\\n                    f.is_symlink() else 'deleted',\n                    # TODO record before and after state for diff-like use\n                    # cases\n                    type=to_state_r['type'],\n                )\n            state = props.get('state', None)\n            if eval_state == 'global' and \\\n                    state not in ('clean', None):\n                # any modification means globally 'modified'\n                return 'modified'\n            if state in ('clean', 'added', 'modified'):\n                props['gitshasum'] = to_state_r['gitshasum']\n                if 'bytesize' in to_state_r:\n                    # if we got this cheap, report it\n                    props['bytesize'] = to_state_r['bytesize']\n                elif props['state'] == 'clean' and 'bytesize' in from_state[f]:\n                    # no change, we can take this old size info\n                    props['bytesize'] = from_state[f]['bytesize']\n            if state in ('clean', 'modified', 'deleted'):\n                props['prev_gitshasum'] = from_state[f]['gitshasum']\n            status[f] = props\n\n        for f, from_state_r in from_state.items():\n            if f not in to_state:\n                # we new this, but now it is gone and Git is not complaining\n                # about it being missing -> properly deleted and deletion\n                # stages\n                status[f] = dict(\n                    state='deleted',\n                    type=from_state_r['type'],\n                    # report the shasum to distinguish from a plainly vanished\n                    # file\n                    gitshasum=from_state_r['gitshasum'],\n                )\n                if eval_state == 'global':\n                    return 'modified'\n\n        if to is not None or eval_state == 'no':\n            # if we have `to` we are specifically comparing against\n            # a recorded state, and this function only attempts\n            # to label the state of a subdataset, not investigate\n            # specifically what the changes in subdatasets are\n            # this is done by a high-level command like rev-diff\n            # so the comparison within this repo and the present\n            # `state` label are all we need, and they are done already\n            if eval_state == 'global':\n                return 'clean'\n            else:\n                return status\n\n        # loop over all subdatasets and look for additional modifications\n        for f, st in status.items():\n            f = str(f)\n            if 'state' in st or not st['type'] == 'dataset':\n                # no business here\n                continue\n            if not GitRepo.is_valid_repo(f):\n                # submodule is not present, no chance for a conflict\n                st['state'] = 'clean'\n                continue\n            # we have to recurse into the dataset and get its status\n            subrepo = GitRepo(f)\n            subrepo_commit = subrepo.get_hexsha()\n            st['gitshasum'] = subrepo_commit\n            # subdataset records must be labeled clean up to this point\n            # test if current commit in subdataset deviates from what is\n            # recorded in the dataset\n            st['state'] = 'modified' \\\n                if st['prev_gitshasum'] != subrepo_commit \\\n                else 'clean'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n            if eval_state == 'commit':\n                continue\n            # the recorded commit did not change, so we need to make\n            # a more expensive traversal\n            st['state'] = subrepo._diffstatus(\n                # we can use 'HEAD' because we know that the commit\n                # did not change. using 'HEAD' will facilitate\n                # caching the result\n                fr='HEAD',\n                to=None,\n                paths=None,\n                untracked=untracked,\n                eval_state='global',\n                eval_file_type=False,\n                _cache=_cache) if st['state'] == 'clean' else 'modified'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n\n        if eval_state == 'global':\n            return 'clean'\n        else:\n            return status\n\n    def _save_pre(self, paths, _status, **kwargs):\n        # helper to get an actionable status report\n        if paths is not None and not paths and not _status:\n            return\n        if _status is None:\n            if 'untracked' not in kwargs:\n                kwargs['untracked'] = 'normal'\n            status = self.status(\n                paths=paths,\n                **{k: kwargs[k] for k in kwargs\n                   if k in ('untracked', 'eval_submodule_state')})\n        else:\n            # we want to be able to add items down the line\n            # make sure to detach from prev. owner\n            status = _status.copy()\n        status = OrderedDict(\n            (k, v) for k, v in status.items()\n            if v.get('state', None) != 'clean'\n        )\n        return status\n\n    def get_staged_paths(self):\n        \"\"\"Returns a list of any stage repository path(s)\n\n        This is a rather fast call, as it will not depend on what is going on\n        in the worktree.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                ['git', 'diff', '--name-only', '--staged'],\n                cwd=self.path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                expect_fail=True)\n        except CommandError as e:\n            lgr.debug(exc_str(e))\n            stdout = ''\n        return [f for f in stdout.split('\\n') if f]\n\n    def _save_post(self, message, status, partial_commit):\n        # helper to commit changes reported in status\n\n        # TODO remove pathobj stringification when commit() can\n        # handle it\n        to_commit = [str(f.relative_to(self.pathobj))\n                     for f, props in status.items()] \\\n                    if partial_commit else None\n        if not partial_commit or to_commit:\n            # we directly call GitRepo.commit() to avoid a whole slew\n            # if direct-mode safeguards and workarounds in the AnnexRepo\n            # implementation (which also run an additional dry-run commit\n            GitRepo.commit(\n                self,\n                files=to_commit,\n                msg=message,\n                options=None,\n                # do not raise on empty commit\n                # it could be that the `add` in this save-cycle has already\n                # brought back a 'modified' file into a clean state\n                careless=True,\n            )\n\n    def save(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Save dataset content.\n\n        Parameters\n        ----------\n        message : str or None\n          A message to accompany the changeset in the log. If None,\n          a default message is used.\n        paths : list or None\n          Any content with path matching any of the paths given in this\n          list will be saved. Matching will be performed against the\n          dataset status (GitRepo.status()), or a custom status provided\n          via `_status`. If no paths are provided, ALL non-clean paths\n          present in the repo status or `_status` will be saved.\n        _status : dict or None\n          If None, Repo.status() will be queried for the given `ds`. If\n          a dict is given, its content will be used as a constraint.\n          For example, to save only modified content, but no untracked\n          content, set `paths` to None and provide a `_status` that has\n          no entries for untracked content.\n        **kwargs :\n          Additional arguments that are passed to underlying Repo methods.\n          Supported:\n\n          - git : bool (passed to Repo.add()\n          - eval_submodule_state : {'full', 'commit', 'no'}\n            passed to Repo.status()\n          - untracked : {'no', 'normal', 'all'} - passed to Repo.satus()\n        \"\"\"\n        return list(\n            self.save_(\n                message=message,\n                paths=paths,\n                _status=_status,\n                **kwargs\n            )\n        )\n\n    def save_(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Like `save()` but working as a generator.\"\"\"\n        from datalad.interface.results import get_status_dict\n\n        status = self._save_pre(paths, _status, **kwargs)\n        if not status:\n            # all clean, nothing todo\n            lgr.debug('Nothing to save in %r, exiting early', self)\n            return\n\n        # three things are to be done:\n        # - remove (deleted if not already staged)\n        # - add (modified/untracked)\n        # - commit (with all paths that have been touched, to bypass\n        #   potential pre-staged bits)\n\n        need_partial_commit = True if self.get_staged_paths() else False\n\n        # remove first, because removal of a subds would cause a\n        # modification of .gitmodules to be added to the todo list\n        to_remove = [\n            # TODO remove pathobj stringification when delete() can\n            # handle it\n            str(f.relative_to(self.pathobj))\n            for f, props in status.items()\n            if props.get('state', None) == 'deleted' and\n            # staged deletions have a gitshasum reported for them\n            # those should not be processed as git rm will error\n            # due to them being properly gone already\n            not props.get('gitshasum', None)]\n        vanished_subds = any(\n            props.get('type', None) == 'dataset' and\n            props.get('state', None) == 'deleted'\n            for f, props in status.items())\n        if to_remove:\n            for r in self.remove(\n                    to_remove,\n                    # we would always see individual files\n                    recursive=False):\n                # TODO normalize result\n                yield get_status_dict(\n                    action='delete',\n                    refds=self.pathobj,\n                    # TODO make remove() report the type\n                    # for now it claims to report on files only\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r)),\n                    # make remove() report on failures too\n                    status='ok',\n                    logger=lgr)\n\n        # TODO this additonal query should not be, base on status as given\n        # if anyhow possible, however, when paths are given, status may\n        # not contain all required information. In case of path=None AND\n        # _status=None, we should be able to avoid this, because\n        # status should have the full info already\n        # looks for contained repositories\n        added_submodule = False\n        untracked_dirs = [f.relative_to(self.pathobj)\n                          for f, props in status.items()\n                          if props.get('state', None) == 'untracked' and\n                          props.get('type', None) == 'directory']\n        to_add_submodules = []\n        if untracked_dirs:\n            to_add_submodules = [sm for sm, sm_props in\n                self.get_content_info(\n                    untracked_dirs,\n                    ref=None,\n                    # request exhaustive list, so that everything that is\n                    # still reported as a directory must be its own repository\n                    untracked='all').items()\n                if sm_props.get('type', None) == 'directory']\n            to_add_submodules = _prune_deeper_repos(to_add_submodules)\n            for cand_sm in to_add_submodules:\n                branch = self.get_active_branch()\n                adjusted_match = ADJUSTED_BRANCH_EXPR.match(\n                    branch if branch else '')\n                try:\n                    self.add_submodule(\n                        str(cand_sm.relative_to(self.pathobj)),\n                        url=None,\n                        name=None,\n                        branch=adjusted_match.group('name') if adjusted_match\n                        else branch\n                    )\n                except (CommandError, InvalidGitRepositoryError) as e:\n                    yield get_status_dict(\n                        action='add_submodule',\n                        ds=self,\n                        path=self.pathobj / ut.PurePosixPath(cand_sm),\n                        status='error',\n                        message=e.stderr if hasattr(e, 'stderr')\n                        else ('not a Git repository: %s', exc_str(e)),\n                        logger=lgr)\n                    continue\n                # This mirrors the result structure yielded for\n                # to_stage_submodules below.\n                yield get_status_dict(\n                    action='add',\n                    refds=self.pathobj,\n                    type='file',\n                    key=None,\n                    path=self.pathobj / ut.PurePosixPath(cand_sm),\n                    status='ok',\n                    logger=lgr)\n                added_submodule = True\n        if not need_partial_commit:\n            # without a partial commit an AnnexRepo would ignore any submodule\n            # path in its add helper, hence `git add` them explicitly\n            to_stage_submodules = {\n                str(f.relative_to(self.pathobj)): props\n                for f, props in status.items()\n                if props.get('state', None) in ('modified', 'untracked')\n                and props.get('type', None) == 'dataset'}\n            if to_stage_submodules:\n                lgr.debug(\n                    '%i submodule path(s) to stage in %r %s',\n                    len(to_stage_submodules), self,\n                    to_stage_submodules\n                    if len(to_stage_submodules) < 10 else '')\n                for r in GitRepo._save_add(\n                        self,\n                        to_stage_submodules,\n                        git_opts=None):\n                    yield r\n\n        if added_submodule or vanished_subds:\n            # need to include .gitmodules in what needs saving\n            status[self.pathobj.joinpath('.gitmodules')] = dict(\n                type='file', state='modified')\n            if hasattr(self, 'annexstatus') and not kwargs.get('git', False):\n                # we cannot simply hook into the coming add-call\n                # as this would go to annex, so make a dedicted git-add\n                # call to ensure .gitmodules is not annexed\n                # in any normal DataLad dataset .gitattributes will\n                # prevent this, but in a plain repo it won't\n                # https://github.com/datalad/datalad/issues/3306\n                for r in GitRepo._save_add(\n                        self,\n                        {op.join(self.path, '.gitmodules'): None}):\n                    yield r\n        to_add = {\n            # TODO remove pathobj stringification when add() can\n            # handle it\n            str(f.relative_to(self.pathobj)): props\n            for f, props in status.items()\n            if (props.get('state', None) in ('modified', 'untracked') and\n                f not in to_add_submodules)}\n        if to_add:\n            lgr.debug(\n                '%i path(s) to add to %s %s',\n                len(to_add), self, to_add if len(to_add) < 10 else '')\n            for r in self._save_add(\n                    to_add,\n                    git_opts=None,\n                    **{k: kwargs[k] for k in kwargs\n                       if k in (('git',) if hasattr(self, 'annexstatus')\n                                else tuple())}):\n                yield r\n\n        self._save_post(message, status, need_partial_commit)\n        # TODO yield result for commit, prev helper checked hexsha pre\n        # and post...\n\n    def _save_add(self, files, git_opts=None):\n        \"\"\"Simple helper to add files in save()\"\"\"\n        from datalad.interface.results import get_status_dict\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                list(files.keys()),\n                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']\n            )\n            # get all the entries\n            for r in self._process_git_get_output(*add_out):\n                yield get_status_dict(\n                    action=r.get('command', 'add'),\n                    refds=self.pathobj,\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r['file']))\n                    if 'file' in r else None,\n                    status='ok' if r.get('success', None) else 'error',\n                    key=r.get('key', None),\n                    # while there is no git-annex underneath here, we\n                    # tend to fake its behavior, so we can also support\n                    # this type of messaging\n                    message='\\n'.join(r['error-messages'])\n                    if 'error-messages' in r else None,\n                    logger=lgr)\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n\n# TODO\n# remove submodule: nope, this is just deinit_submodule + remove\n# status?\n\n\ndef _fixup_submodule_dotgit_setup(ds, relativepath):\n    \"\"\"Implementation of our current of .git in a subdataset\n\n    Each subdataset/module has its own .git directory where a standalone\n    repository would have it. No gitdir files, no symlinks.\n    \"\"\"\n    # move .git to superrepo's .git/modules, remove .git, create\n    # .git-file\n    path = opj(ds.path, relativepath)\n    subds_dotgit = opj(path, \".git\")\n\n    repo = GitRepo(path, create=False)\n    if repo.dot_git.parent == repo.pathobj:\n        # this is what we want\n        return\n\n    # first we want to remove any conflicting worktree setup\n    # done by git to find the checkout at the mountpoint of the\n    # submodule, if we keep that, any git command will fail\n    # after we move .git\n    # Ben: Shouldn't we re-setup a possible worktree afterwards?\n    repo.config.unset('core.worktree', where='local')\n    # what we have here is some kind of reference, remove and\n    # replace by the target\n    os.remove(subds_dotgit)\n    # make absolute\n    src_dotgit = str(repo.dot_git)\n    # move .git\n    from os import rename, listdir, rmdir\n    ensure_dir(subds_dotgit)\n    for dot_git_entry in listdir(src_dotgit):\n        rename(opj(src_dotgit, dot_git_entry),\n               opj(subds_dotgit, dot_git_entry))\n    assert not listdir(src_dotgit)\n    rmdir(src_dotgit)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to Git via GitPython\n\nFor further information on GitPython see http://gitpython.readthedocs.org/\n\n\"\"\"\n\nimport re\nimport time\nimport os\nimport os.path as op\nimport warnings\nfrom locale import getpreferredencoding\n\n\nfrom itertools import chain\nimport logging\nfrom collections import (\n    OrderedDict,\n    namedtuple\n)\nfrom os import linesep\nfrom os.path import (\n    join as opj,\n    exists,\n    normpath,\n    isabs,\n    commonprefix,\n    relpath,\n    realpath,\n    dirname,\n    basename,\n    curdir,\n    pardir,\n    sep\n)\n\nimport posixpath\nfrom functools import wraps\nfrom weakref import WeakValueDictionary\n\nimport git as gitpy\nfrom gitdb.exc import BadName\nfrom git.exc import (\n    GitCommandError,\n    NoSuchPathError,\n    InvalidGitRepositoryError\n)\nfrom datalad.log import log_progress\nfrom datalad.support.due import due, Doi\n\nfrom datalad import ssh_manager\nfrom datalad.cmd import (\n    WitlessRunner,\n    WitlessProtocol,\n    GitRunner,\n    BatchedCommand,\n    run_gitcommand_on_file_list_chunks,\n    StdOutErrCapture,\n)\nfrom datalad.config import (\n    ConfigManager,\n    _parse_gitconfig_dump\n)\n\nfrom datalad.consts import (\n    ADJUSTED_BRANCH_EXPR,\n)\nfrom datalad.dochelpers import exc_str\nimport datalad.utils as ut\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    ensure_list,\n    optional_args,\n    on_windows,\n    getpwd,\n    posix_relpath,\n    ensure_dir,\n    generate_file_chunks,\n    ensure_unicode,\n    split_cmdline,\n)\n\n# imports from same module:\nfrom .external_versions import external_versions\nfrom .exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    GitIgnoreError,\n    InvalidGitReferenceError,\n    MissingBranchError,\n    OutdatedExternalDependencyWarning,\n    PathKnownToRepositoryError,\n)\nfrom .network import (\n    RI,\n    PathRI,\n    is_ssh\n)\nfrom .path import get_parent_paths\nfrom .repo import (\n    PathBasedFlyweight,\n    RepoInterface\n)\n\n#from git.objects.blob import Blob\n\n# shortcuts\n_curdirsep = curdir + sep\n_pardirsep = pardir + sep\n\n\nlgr = logging.getLogger('datalad.gitrepo')\n_lgr_level = lgr.getEffectiveLevel()\nif _lgr_level <= 2:\n    from ..log import LoggerHelper\n    # Let's also enable gitpy etc debugging\n    gitpy_lgr = LoggerHelper(logtarget=\"git\").get_initialized_logger()\n    gitpy_lgr.setLevel(_lgr_level)\n    gitpy_lgr.propagate = True\n\n# Override default GitPython's DB backend to talk directly to git so it doesn't\n# interfere with possible operations performed by gc/repack\ndefault_git_odbt = gitpy.GitCmdObjectDB\n\n# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used.\n# Didn't work as expected on a first try. Probably there is a neatier way to\n# log Exceptions from git commands.\n\n\ndef to_options(split_single_char_options=True, **kwargs):\n    \"\"\"Transform keyword arguments into a list of cmdline options\n\n    Imported from GitPython.\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n\n    Parameters\n    ----------\n    split_single_char_options: bool\n\n    kwargs:\n\n    Returns\n    -------\n    list\n    \"\"\"\n    def dashify(string):\n        return string.replace('_', '-')\n\n    def transform_kwarg(name, value, split_single_char_options):\n        if len(name) == 1:\n            if value is True:\n                return [\"-%s\" % name]\n            elif value not in (False, None):\n                if split_single_char_options:\n                    return [\"-%s\" % name, \"%s\" % value]\n                else:\n                    return [\"-%s%s\" % (name, value)]\n        else:\n            if value is True:\n                return [\"--%s\" % dashify(name)]\n            elif value is not False and value is not None:\n                return [\"--%s=%s\" % (dashify(name), value)]\n        return []\n\n    args = []\n    kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0]))\n    for k, v in kwargs.items():\n        if isinstance(v, (list, tuple)):\n            for value in v:\n                args += transform_kwarg(k, value, split_single_char_options)\n        else:\n            args += transform_kwarg(k, v, split_single_char_options)\n    return args\n\n\ndef _normalize_path(base_dir, path):\n    \"\"\"Helper to check paths passed to methods of this class.\n\n    Checks whether `path` is beneath `base_dir` and normalizes it.\n    Additionally paths are converted into relative paths with respect to\n    `base_dir`, considering PWD in case of relative paths. This\n    is intended to be used in repository classes, which means that\n    `base_dir` usually will be the repository's base directory.\n\n    Parameters\n    ----------\n    base_dir: str\n        directory to serve as base to normalized, relative paths\n    path: str\n        path to be normalized\n\n    Returns\n    -------\n    str:\n        path, that is a relative path with respect to `base_dir`\n    \"\"\"\n    if not path:\n        return path\n\n    base_dir = realpath(base_dir)  # realpath OK\n    # path = normpath(path)\n    # Note: disabled normpath, because it may break paths containing symlinks;\n    # But we don't want to realpath relative paths, in case cwd isn't the\n    # correct base.\n\n    if isabs(path):\n        # path might already be a symlink pointing to annex etc,\n        # so realpath only its directory, to get \"inline\" with\n        # realpath(base_dir) above\n        path = opj(realpath(dirname(path)), basename(path))  # realpath OK\n    # Executive decision was made to not do this kind of magic!\n    #\n    # elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:\n    #     # If we are inside repository, rebuilt relative paths.\n    #     path = opj(realpath(getpwd()), path)\n    #\n    # BUT with relative curdir/pardir start it would assume relative to curdir\n    #\n    elif path.startswith(_curdirsep) or path.startswith(_pardirsep):\n        path = normpath(opj(realpath(getpwd()), path))  # realpath OK\n    else:\n        # We were called from outside the repo. Therefore relative paths\n        # are interpreted as being relative to self.path already.\n        return path\n\n    if commonprefix([path, base_dir]) != base_dir:\n        raise FileNotInRepositoryError(msg=\"Path outside repository: %s\"\n                                           % path, filename=path)\n\n    return relpath(path, start=base_dir)\n\n\n@optional_args\ndef normalize_path(func):\n    \"\"\"Decorator to provide unified path conversion for a single file\n\n    Unlike normalize_paths, intended to be used for functions dealing with a\n    single filename at a time\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, file_, *args, **kwargs):\n        file_new = _normalize_path(self.path, file_)\n        return func(self, file_new, *args, **kwargs)\n\n    return newfunc\n\n\n@optional_args\ndef normalize_paths(func, match_return_type=True, map_filenames_back=False,\n                    serialize=False):\n    \"\"\"Decorator to provide unified path conversions.\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path or a list of paths at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n\n    Accepts either a list of paths or a single path in a str. Passes a list\n    to decorated function either way, but would return based on the value of\n    match_return_type and possibly input argument.\n\n    If a call to the wrapped function includes normalize_path and it is False\n    no normalization happens for that function call (used for calls to wrapped\n    functions within wrapped functions, while possible CWD is within a\n    repository)\n\n    Parameters\n    ----------\n    match_return_type : bool, optional\n      If True, and a single string was passed in, it would return the first\n      element of the output (after verifying that it is a list of length 1).\n      It makes easier to work with single files input.\n    map_filenames_back : bool, optional\n      If True and returned value is a dictionary, it assumes to carry entries\n      one per file, and then filenames are mapped back to as provided from the\n      normalized (from the root of the repo) paths\n    serialize : bool, optional\n      Loop through files giving only a single one to the function one at a time.\n      This allows to simplify implementation and interface to annex commands\n      which do not take multiple args in the same call (e.g. checkpresentkey)\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, files, *args, **kwargs):\n\n        normalize = _normalize_path if kwargs.pop('normalize_paths', True) \\\n            else lambda rpath, filepath: filepath\n\n        if files:\n            if isinstance(files, str) or not files:\n                files_new = [normalize(self.path, files)]\n                single_file = True\n            elif isinstance(files, list):\n                files_new = [normalize(self.path, path) for path in files]\n                single_file = False\n            else:\n                raise ValueError(\"_files_decorator: Don't know how to handle \"\n                                 \"instance of %s.\" % type(files))\n        else:\n            single_file = None\n            files_new = []\n\n        if map_filenames_back:\n            def remap_filenames(out):\n                \"\"\"Helper to map files back to non-normalized paths\"\"\"\n                if isinstance(out, dict):\n                    assert(len(out) == len(files_new))\n                    files_ = [files] if single_file else files\n                    mapped = out.__class__()\n                    for fin, fout in zip(files_, files_new):\n                        mapped[fin] = out[fout]\n                    return mapped\n                else:\n                    return out\n        else:\n            remap_filenames = lambda x: x\n\n        if serialize:  # and not single_file:\n            result = [\n                func(self, f, *args, **kwargs)\n                for f in files_new\n            ]\n        else:\n            result = func(self, files_new, *args, **kwargs)\n\n        if single_file is None:\n            # no files were provided, nothing we can do really\n            return result\n        elif (result is None) or not match_return_type or not single_file:\n            # If function doesn't return anything or no denormalization\n            # was requested or it was not a single file\n            return remap_filenames(result)\n        elif single_file:\n            if len(result) != 1:\n                # Magic doesn't apply\n                return remap_filenames(result)\n            elif isinstance(result, (list, tuple)):\n                return result[0]\n            elif isinstance(result, dict) and tuple(result)[0] == files_new[0]:\n                # assume that returned dictionary has files as keys.\n                return tuple(result.values())[0]\n            else:\n                # no magic can apply\n                return remap_filenames(result)\n        else:\n            return RuntimeError(\"should have not got here... check logic\")\n\n    return newfunc\n\n\ndef _remove_empty_items(list_):\n    \"\"\"Remove empty entries from list\n\n    This is needed, since some functions of GitPython may convert\n    an empty entry to '.', when used with a list of paths.\n\n    Parameter:\n    ----------\n    list_: list of str\n\n    Returns\n    -------\n    list of str\n    \"\"\"\n    if not isinstance(list_, list):\n        lgr.warning(\n            \"_remove_empty_items() called with non-list type: %s\" % type(list_))\n        return list_\n    return [file_ for file_ in list_ if file_]\n\n\nif \"2.24.0\" <= external_versions[\"cmd:git\"] < \"2.25.0\":\n    # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing\n    # into untracked submodules when multiple pathspecs are given, returning\n    # repositories that are deeper than the first level. This helper filters\n    # these deeper levels out so that save_() doesn't fail trying to add them.\n    #\n    # This regression fixed with upstream's 072a231016 (2019-12-10).\n    def _prune_deeper_repos(repos):\n        firstlevel_repos = []\n        prev = None\n        for repo in sorted(repos):\n            if not (prev and str(repo).startswith(prev)):\n                prev = str(repo)\n                firstlevel_repos.append(repo)\n        return firstlevel_repos\nelse:\n    def _prune_deeper_repos(repos):\n        return repos\n\n\ndef Repo(*args, **kwargs):\n    \"\"\"Factory method around gitpy.Repo to consistently initiate with different\n    backend\n    \"\"\"\n    # TODO: This probably doesn't work as intended (or at least not as\n    #       consistently as intended). gitpy.Repo could be instantiated by\n    #       classmethods Repo.init or Repo.clone_from. In these cases 'odbt'\n    #       would be needed as a parameter to these methods instead of the\n    #       constructor.\n    if 'odbt' not in kwargs:\n        kwargs['odbt'] = default_git_odbt\n    return gitpy.Repo(*args, **kwargs)\n\n\ndef guard_BadName(func):\n    \"\"\"A helper to guard against BadName exception\n\n    Workaround for\n    https://github.com/gitpython-developers/GitPython/issues/768\n    also see https://github.com/datalad/datalad/issues/2550\n    Let's try to precommit (to flush anything flushable) and do\n    it again\n    \"\"\"\n\n    @wraps(func)\n    def wrapped(repo, *args, **kwargs):\n        try:\n            return func(repo, *args, **kwargs)\n        except BadName:\n            repo.precommit()\n            return func(repo, *args, **kwargs)\n\n    return wrapped\n\n\nclass GitProgress(WitlessProtocol):\n    \"\"\"Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    # inform super-class to capture stderr\n    proc_err = True\n\n    _num_op_codes = 10\n    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \\\n        [1 << x for x in range(_num_op_codes)]\n    STAGE_MASK = BEGIN | END\n    OP_MASK = ~STAGE_MASK\n\n    DONE_TOKEN = 'done.'\n    TOKEN_SEPARATOR = ', '\n\n    _known_ops = {\n        COUNTING: (\"Counting\", \"Objects\"),\n        ENUMERATING: (\"Enumerating\", \"Objects\"),\n        COMPRESSING: (\"Compressing\", \"Objects\"),\n        WRITING: (\"Writing\", \"Objects\"),\n        RECEIVING: (\"Receiving\", \"Objects\"),\n        RESOLVING: (\"Resolving\", \"Deltas\"),\n        FINDING_SOURCES: (\"Finding\", \"Sources\"),\n        CHECKING_OUT: (\"Check out\", \"Things\"),\n    }\n\n    __slots__ = ('_seen_ops', '_pbars', '_encoding')\n\n    re_op_absolute = re.compile(r\"(remote: )?([\\w\\s]+):\\s+()(\\d+)()(.*)\")\n    re_op_relative = re.compile(r\"(remote: )?([\\w\\s]+):\\s+(\\d+)% \\((\\d+)/(\\d+)\\)(.*)\")\n\n    def __init__(self, *args):\n        super().__init__(*args)\n        self._encoding = getpreferredencoding(do_setlocale=False)\n        self._unprocessed = None\n\n    def connection_made(self, transport):\n        super().connection_made(transport)\n        self._seen_ops = []\n        self._pbars = set()\n\n    def process_exited(self):\n        # take down any progress bars that were not closed orderly\n        for pbar_id in self._pbars:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished',\n            )\n        super().process_exited()\n\n    def pipe_data_received(self, fd, byts):\n        # progress reports only come from stderr\n        if fd != 2:\n            # let the base class decide what to do with it\n            super().pipe_data_received(fd, byts)\n            return\n        for line in byts.splitlines(keepends=True):\n            # put any unprocessed content back in front\n            line = self._unprocessed + line if self._unprocessed else line\n            self._unprocessed = None\n            if not self._parse_progress_line(line):\n                # anything that doesn't look like a progress report\n                # is retained and returned\n                # in case of partial progress lines, this can lead to\n                # leakage of progress info into the output, but\n                # it is better to enable better (maybe more expensive)\n                # subsequent filtering than hidding lines with\n                # unknown, potentially important info\n                lgr.debug('Non-progress stderr: %s', line)\n                if line.endswith((b'\\r', b'\\n')):\n                    # complete non-progress line, pass on\n                    super().pipe_data_received(fd, line)\n                else:\n                    # an incomplete line, maybe the next batch completes\n                    # it to become a recognizable progress report\n                    self._unprocessed = line\n\n    def _parse_progress_line(self, line):\n        \"\"\"Process a single line\n\n        Parameters\n        ----------\n        line : bytes\n\n        Returns\n        -------\n        bool\n          Flag whether the line was recognized as a Git progress report.\n        \"\"\"\n        # handle\n        # Counting objects: 4, done.\n        # Compressing objects:  50% (1/2)\n        # Compressing objects: 100% (2/2)\n        # Compressing objects: 100% (2/2), done.\n        line = line.decode(self._encoding) if isinstance(line, bytes) else line\n        if line.startswith(('warning:', 'error:', 'fatal:')):\n            return False\n\n        # find escape characters and cut them away - regex will not work with\n        # them as they are non-ascii. As git might expect a tty, it will send them\n        last_valid_index = None\n        for i, c in enumerate(reversed(line)):\n            if ord(c) < 32:\n                # its a slice index\n                last_valid_index = -i - 1\n            # END character was non-ascii\n        # END for each character in line\n        if last_valid_index is not None:\n            line = line[:last_valid_index]\n        # END cut away invalid part\n        line = line.rstrip()\n\n        cur_count, max_count = None, None\n        match = self.re_op_relative.match(line)\n        if match is None:\n            match = self.re_op_absolute.match(line)\n\n        if not match:\n            return False\n        # END could not get match\n\n        op_code = 0\n        _remote, op_name, _percent, cur_count, max_count, message = match.groups()\n\n        # get operation id\n        if op_name == \"Counting objects\":\n            op_code |= self.COUNTING\n        elif op_name == \"Compressing objects\":\n            op_code |= self.COMPRESSING\n        elif op_name == \"Writing objects\":\n            op_code |= self.WRITING\n        elif op_name == 'Receiving objects':\n            op_code |= self.RECEIVING\n        elif op_name == 'Resolving deltas':\n            op_code |= self.RESOLVING\n        elif op_name == 'Finding sources':\n            op_code |= self.FINDING_SOURCES\n        elif op_name == 'Checking out files':\n            op_code |= self.CHECKING_OUT\n        elif op_name == 'Enumerating objects':\n            op_code |= self.ENUMERATING\n        else:\n            # Note: On windows it can happen that partial lines are sent\n            # Hence we get something like \"CompreReceiving objects\", which is\n            # a blend of \"Compressing objects\" and \"Receiving objects\".\n            # This can't really be prevented.\n            lgr.debug(\n                'Output line matched a progress report of an unknown type: %s',\n                line)\n            # TODO investigate if there is any chance that we might swallow\n            # important info -- until them do not flag this line\n            # as progress\n            return False\n        # END handle op code\n\n        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)\n\n        op_props = self._known_ops[op_code]\n\n        # figure out stage\n        if op_code not in self._seen_ops:\n            self._seen_ops.append(op_code)\n            op_code |= self.BEGIN\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n                label=op_props[0],\n                unit=' {}'.format(op_props[1]),\n                total=float(max_count) if max_count else None,\n            )\n            self._pbars.add(pbar_id)\n        # END begin opcode\n\n        if message is None:\n            message = ''\n        # END message handling\n\n        done_progress = False\n        message = message.strip()\n        if message.endswith(self.DONE_TOKEN):\n            op_code |= self.END\n            message = message[:-len(self.DONE_TOKEN)]\n            done_progress = True\n        # END end message handling\n        message = message.strip(self.TOKEN_SEPARATOR)\n\n        if cur_count and max_count:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                message,\n                update=float(cur_count),\n            )\n\n        if done_progress:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n            )\n            self._pbars.discard(pbar_id)\n        return True\n\n\nclass StdOutCaptureWithGitProgress(GitProgress):\n    proc_out = True\n\n\nclass FetchInfo(dict):\n    \"\"\"\n    dict that carries results of a fetch operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n\n    NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \\\n        FAST_FORWARD, ERROR = [1 << x for x in range(8)]\n\n    _re_fetch_result = re.compile(r'^\\s*(.) (\\[?[\\w\\s\\.$@]+\\]?)\\s+(.+) [-> ]+ ([^\\s]+)(    \\(.*\\)?$)?')\n\n    _flag_map = {\n        '!': ERROR,\n        '+': FORCED_UPDATE,\n        '*': 0,\n        '=': HEAD_UPTODATE,\n        ' ': FAST_FORWARD,\n        '-': TAG_UPDATE,\n    }\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        HEAD_UPTODATE: 'uptodate',\n        TAG_UPDATE: 'tag-update',\n        REJECTED: 'rejected',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Parse information from the given line as returned by git-fetch -v\n        and return a new FetchInfo object representing this information.\n        \"\"\"\n        match = cls._re_fetch_result.match(line)\n        if match is None:\n            raise ValueError(\"Failed to parse line: %r\" % line)\n\n        # parse lines\n        control_character, operation, local_remote_ref, remote_local_ref, note = \\\n            match.groups()\n\n        # parse flags from control_character\n        flags = 0\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\n                \"Control character %r unknown as parsed from line %r\"\n                % (control_character, line))\n        # END control char exception handling\n\n        # parse operation string for more info - makes no sense for symbolic refs,\n        # but we parse it anyway\n        old_commit = None\n        if 'rejected' in operation:\n            flags |= cls.REJECTED\n        if 'new tag' in operation:\n            flags |= cls.NEW_TAG\n        if 'tag update' in operation:\n            flags |= cls.TAG_UPDATE\n        if 'new branch' in operation:\n            flags |= cls.NEW_HEAD\n        if '...' in operation or '..' in operation:\n            split_token = '...'\n            if control_character == ' ':\n                split_token = split_token[:-1]\n            old_commit = operation.split(split_token)[0]\n        # END handle refspec\n\n        return cls(\n            ref=remote_local_ref.strip(),\n            local_ref=local_remote_ref.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=note,\n            old_commit=old_commit,\n        )\n\n\nclass PushInfo(dict):\n    \"\"\"dict that carries results of a push operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    NEW_TAG, NEW_HEAD, NO_MATCH, REJECTED, REMOTE_REJECTED, REMOTE_FAILURE, DELETED, \\\n        FORCED_UPDATE, FAST_FORWARD, UP_TO_DATE, ERROR = [1 << x for x in range(11)]\n\n    _flag_map = {'X': NO_MATCH,\n                 '-': DELETED,\n                 '*': 0,\n                 '+': FORCED_UPDATE,\n                 ' ': FAST_FORWARD,\n                 '=': UP_TO_DATE,\n                 '!': ERROR}\n\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        NO_MATCH: 'no-match',\n        REJECTED: 'rejected',\n        REMOTE_REJECTED: 'remote-rejected',\n        REMOTE_FAILURE: 'remote-failure',\n        DELETED: 'deleted',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        UP_TO_DATE: 'uptodate',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Create a new PushInfo instance as parsed from line which is expected to be like\n            refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes\"\"\"\n        control_character, from_to, summary = line.split('\\t', 3)\n        flags = 0\n\n        # control character handling\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\"Control character %r unknown as parsed from line %r\" % (control_character, line))\n        # END handle control character\n\n        # from_to handling\n        from_ref_string, to_ref_string = from_to.split(':')\n\n        # commit handling, could be message or commit info\n        old_commit = None\n        if summary.startswith('['):\n            if \"[rejected]\" in summary:\n                flags |= cls.REJECTED\n            elif \"[remote rejected]\" in summary:\n                flags |= cls.REMOTE_REJECTED\n            elif \"[remote failure]\" in summary:\n                flags |= cls.REMOTE_FAILURE\n            elif \"[no match]\" in summary:\n                flags |= cls.ERROR\n            elif \"[new tag]\" in summary:\n                flags |= cls.NEW_TAG\n            elif \"[new branch]\" in summary:\n                flags |= cls.NEW_HEAD\n            # uptodate encoded in control character\n        else:\n            # fast-forward or forced update - was encoded in control character,\n            # but we parse the old and new commit\n            split_token = \"...\"\n            if control_character == \" \":\n                split_token = \"..\"\n            old_sha, _new_sha = summary.split(' ')[0].split(split_token)\n            # have to use constructor here as the sha usually is abbreviated\n            old_commit = old_sha\n        # END message handling\n\n        return cls(\n            from_ref=from_ref_string.strip(),\n            to_ref=to_ref_string.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=summary.strip(),\n            old_commit=old_commit,\n        )\n\n\n# Compatibility kludge.  See GitRepo.get_submodules().\nSubmodule = namedtuple(\"Submodule\", [\"name\", \"path\", \"url\"])\n\n\nclass GitRepo(RepoInterface, metaclass=PathBasedFlyweight):\n    \"\"\"Representation of a git repository\n\n    \"\"\"\n    # We must check git config to have name and email set, but\n    # should do it once\n    _config_checked = False\n\n    # Begin Flyweight:\n\n    _unique_instances = WeakValueDictionary()\n\n    def _flyweight_invalid(self):\n        return not self.is_valid_git()\n\n    @classmethod\n    def _flyweight_reject(cls, id_, *args, **kwargs):\n        # TODO:\n        # This is a temporary approach. See PR # ...\n        # create = kwargs.pop('create', None)\n        # kwargs.pop('path', None)\n        # if create and kwargs:\n        #     # we have `create` plus options other than `path`\n        #     return \"Call to {0}() with args {1} and kwargs {2} conflicts \" \\\n        #            \"with existing instance {3}.\" \\\n        #            \"This is likely to be caused by inconsistent logic in \" \\\n        #            \"your code.\" \\\n        #            \"\".format(cls, args, kwargs, cls._unique_instances[id_])\n        pass\n\n    # End Flyweight\n\n    def __hash__(self):\n        # the flyweight key is already determining unique instances\n        # add the class name to distinguish from strings of a path\n        return hash((self.__class__.__name__, self.__weakref__.key))\n\n    # This is the least common denominator to claim that a user\n    # used DataLad.\n    # For now citing Zenodo's all (i.e., latest) version\n    @due.dcite(Doi(\"10.5281/zenodo.808846\"),\n               # override path since there is no need ATM for such details\n               path=\"datalad\",\n               description=\"DataLad - Data management and distribution platform\")\n    def __init__(self, path, url=None, runner=None, create=True,\n                 git_opts=None, repo=None, fake_dates=False,\n                 create_sanity_checks=True,\n                 **kwargs):\n        \"\"\"Creates representation of git repository at `path`.\n\n        Can also be used to create a git repository at `path`.\n\n        Parameters\n        ----------\n        path: str\n          path to the git repository; In case it's not an absolute path,\n          it's relative to PWD\n        url: str, optional\n          DEPRECATED -- use .clone() class method\n          url to the to-be-cloned repository. Requires a valid git url\n          according to:\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        create: bool, optional\n          if true, creates a git repository at `path` if there is none. Also\n          creates `path`, if it doesn't exist.\n          If set to false, an exception is raised in case `path` doesn't exist\n          or doesn't contain a git repository.\n        repo: git.Repo, optional\n          GitPython's Repo instance to (re)use if provided\n        create_sanity_checks: bool, optional\n          Whether to perform sanity checks during initialization (when\n          `create=True` and target path is not a valid repo already), such as\n          that new repository is not created in the directory where git already\n          tracks some files.\n        kwargs:\n          keyword arguments serving as additional options to the git-init\n          command. Therefore, it makes sense only if called with `create`.\n\n          Generally, this way of passing options to the git executable is\n          (or will be) used a lot in this class. It's a transformation of\n          python-style keyword arguments (or a `dict`) to command line arguments,\n          provided by GitPython.\n\n          A single character keyword will be prefixed by '-', multiple characters\n          by '--'. An underscore in the keyword becomes a dash. The value of the\n          keyword argument is used as the value for the corresponding command\n          line argument. Assigning a boolean creates a flag.\n\n          Examples:\n          no_commit=True => --no-commit\n          C='/my/path'   => -C /my/path\n\n        \"\"\"\n\n        # BEGIN Repo validity test\n        # We want to fail early for tests, that would be performed a lot. In particular this is about\n        # GitRepo.is_valid_repo. We would use the latter to decide whether or not to call GitRepo() only for\n        # __init__ to then test the same things again. If we fail early we can save the additional test from outer\n        # scope.\n        self.path = path\n\n        # Note, that the following three path objects are used often and therefore\n        # are stored for performance. Path object creation comes with a cost. Most noteably,\n        # this is used for validity checking of the repository.\n        self.pathobj = ut.Path(self.path)\n        self.dot_git = self.pathobj / '.git'\n        # Read a potential .git file in order to not do that over and over again, when testing is_valid_git() etc.\n        # TODO: There's still some code duplication with static method GitRepo.get_git_dir()\n        #       However, it's returning relative path. So, the logic in usage needs to be unified in order to melt both\n        #       pieces.\n        if self.dot_git.is_file():\n            with self.dot_git.open() as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    self.dot_git = self.pathobj / line[7:].strip()\n                else:\n                    raise InvalidGitRepositoryError(\"Invalid .git file\")\n\n        elif self.dot_git.is_symlink():\n            self.dot_git = self.dot_git.resolve()\n\n        self._valid_git_test_path = self.dot_git / 'HEAD'\n        _valid_repo = self.is_valid_git()\n\n        do_create = False\n        if create and not _valid_repo:\n            if repo is not None:\n                # `repo` passed with `create`, which doesn't make sense\n                raise TypeError(\"argument 'repo' must not be used with 'create'\")\n            do_create = True\n        else:\n            # Note: We used to call gitpy.Repo(path) here, which potentially\n            # raised NoSuchPathError or InvalidGitRepositoryError. This is\n            # used by callers of GitRepo.__init__() to detect whether we have a\n            # valid repo at `path`. Now, with switching to lazy loading property\n            # `repo`, we detect those cases without instantiating a\n            # gitpy.Repo().\n\n            if not exists(path):\n                raise NoSuchPathError(path)\n            if not _valid_repo:\n                raise InvalidGitRepositoryError(path)\n        # END Repo validity test\n\n        if url is not None:\n            raise DeprecatedError(\n                new=\".clone() class method\",\n                version=\"0.5.0\",\n                msg=\"RF: url passed to init()\"\n            )\n\n        # So that we \"share\" control paths with git/git-annex\n        if ssh_manager:\n            ssh_manager.assure_initialized()\n\n        # note: we may also want to distinguish between a path to the worktree\n        # and the actual repository\n\n        # Could be used to e.g. disable automatic garbage and autopacking\n        # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0']\n        self._GIT_COMMON_OPTIONS = []\n        # actually no need with default GitPython db backend not in memory\n        # default_git_odbt but still allows for faster testing etc.\n        # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = []\n\n        if git_opts is None:\n            git_opts = {}\n        if kwargs:\n            git_opts.update(kwargs)\n\n        self.cmd_call_wrapper = runner or GitRunner(cwd=self.path)\n        self._repo = repo\n        self._cfg = None\n\n        if do_create:  # we figured it out ealier\n            self._repo = self._create_empty_repo(path,\n                                                 create_sanity_checks, **git_opts)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # calls above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        # with DryRunProtocol path might still not exist\n        if exists(self.path):\n            self.inode = os.stat(self.path).st_ino\n        else:\n            self.inode = None\n\n        if fake_dates:\n            self.configure_fake_dates()\n        # Set by fake_dates_enabled to cache config value across this instance.\n        self._fake_dates_enabled = None\n\n    def _create_empty_repo(self, path, sanity_checks=True, **kwargs):\n        if not op.lexists(path):\n            os.makedirs(path)\n        elif sanity_checks and external_versions['cmd:git'] < '2.14.0':\n            warnings.warn(\n                \"Your git version (%s) is too old, we will not safe-guard \"\n                \"against creating a new repository under already known to git \"\n                \"subdirectory\" % external_versions['cmd:git'],\n                OutdatedExternalDependencyWarning\n            )\n        elif sanity_checks:\n            # Verify that we are not trying to initialize a new git repository\n            # under a directory some files of which are already tracked by git\n            # use case: https://github.com/datalad/datalad/issues/3068\n            try:\n                stdout, _ = self._git_custom_command(\n                    None, ['git', 'ls-files'], cwd=path, expect_fail=True\n                )\n                if stdout:\n                    raise PathKnownToRepositoryError(\n                        \"Failing to initialize new repository under %s where \"\n                        \"following files are known to a repository above: %s\"\n                        % (path, stdout)\n                    )\n            except CommandError:\n                # assume that all is good -- we are not under any repo\n                pass\n\n        cmd = ['git', 'init']\n        cmd.extend(kwargs.pop('_from_cmdline_', []))\n        cmd.extend(to_options(**kwargs))\n        lgr.debug(\n            \"Initialize empty Git repository at '%s'%s\",\n            path,\n            ' %s' % cmd[2:] if cmd[2:] else '')\n\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                cmd,\n                cwd=path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            lgr.error(exc_str(exc))\n            raise\n        # we want to return None and have lazy eval take care of\n        # the rest\n        return\n\n    @property\n    def repo(self):\n        # with DryRunProtocol path not exist\n        if exists(self.path):\n            inode = os.stat(self.path).st_ino\n        else:\n            inode = None\n        if self.inode != inode:\n\n            # TODO: - what if we never had any inode? possible?\n            #       - also: call to self._repo and only afterwards checking whether it's None seems strange\n\n            # reset background processes invoked by GitPython:\n            # it can be that we never got to instantiate a GitPython repo instance\n            if self._repo:\n                self._repo.git.clear_cache()\n            self.inode = inode\n\n        if self._repo is None:\n            # Note, that this may raise GitCommandError, NoSuchPathError,\n            # InvalidGitRepositoryError:\n            self._repo = self.cmd_call_wrapper(\n                Repo,\n                self.path)\n            lgr.log(8, \"Using existing Git repository at %s\", self.path)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # call of Repo(path) above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        return self._repo\n\n    @classmethod\n    def clone(cls, url, path, *args, clone_options=None, **kwargs):\n        \"\"\"Clone url into path\n\n        Provides workarounds for known issues (e.g.\n        https://github.com/datalad/datalad/issues/785)\n\n        Parameters\n        ----------\n        url : str\n        path : str\n        clone_options : dict\n          Key/value pairs of arbitrary options that will be passed on to the\n          underlying call to `git-clone`.\n        expect_fail : bool\n          Whether expect that command might fail, so error should be logged then\n          at DEBUG level instead of ERROR\n        \"\"\"\n\n        if 'repo' in kwargs:\n            raise TypeError(\"argument 'repo' conflicts with cloning\")\n            # TODO: what about 'create'?\n\n        expect_fail = kwargs.pop('expect_fail', False)\n        # fail early on non-empty target:\n        from os import listdir\n        if exists(path) and listdir(path):\n            # simulate actual GitCommandError:\n            lgr.warning(\"destination path '%s' already exists and is not an \"\n                        \"empty directory.\" % path)\n            raise GitCommandError(\n                ['git', 'clone', '-v', url, path],\n                128,\n                \"fatal: destination path '%s' already exists and is not an \"\n                \"empty directory.\" % path)\n        else:\n            # protect against cloning into existing and obviously dangling\n            # instance for that location\n            try:\n                del cls._unique_instances[path]\n            except KeyError:\n                # didn't exist - all fine\n                pass\n\n        # Massage URL\n        url_ri = RI(url) if not isinstance(url, RI) else url\n        if not on_windows:\n            # if we are on windows, the local path of a URL\n            # would not end up being a proper local path and cloning\n            # would fail. Don't try to be smart and just pass the\n            # URL along unmodified\n\n            # try to get a local path from `url`:\n            try:\n                url = url_ri.localpath\n                url_ri = RI(url)\n            except ValueError:\n                pass\n\n        if is_ssh(url_ri):\n            ssh_manager.get_connection(url).open()\n        else:\n            if isinstance(url_ri, PathRI):\n                # expand user, because execution not going through a shell\n                # doesn't work well otherwise\n                new_url = os.path.expanduser(url)\n                if url != new_url:\n                    lgr.info(\"Expanded source path to %s from %s\", new_url, url)\n                    url = new_url\n\n        fix_annex = None\n        ntries = 5  # 3 is not enough for robust workaround\n        for trial in range(ntries):\n            try:\n                lgr.debug(\"Git clone from {0} to {1}\".format(url, path))\n\n                out, err = WitlessRunner(\n                    env=GitRunner.get_git_environ_adjusted()).run(\n                        ['git', 'clone', '--progress', url, path] \\\n                        + (to_options(**clone_options)\n                           if clone_options else []),\n                        protocol=GitProgress,\n                )\n                # fish out non-critical warnings by git-clone\n                # (empty repo clone, etc.), all other content is logged\n                # by the progress helper to 'debug'\n                for errline in err.splitlines():\n                    if errline.startswith('warning:'):\n                        lgr.warning(errline[8:].strip())\n                lgr.debug(\"Git clone completed\")\n                break\n            except CommandError as e:\n                # log here but let caller decide what to do\n                e_str = exc_str(e)\n                # see https://github.com/datalad/datalad/issues/785\n                if re.search(\"Request for .*aborted.*Unable to find\", str(e),\n                             re.DOTALL) \\\n                        and trial < ntries - 1:\n                    lgr.info(\n                        \"Hit a known issue with Git (see GH#785). Trial #%d, \"\n                        \"retrying\",\n                        trial)\n                    continue\n                    (lgr.debug if expect_fail else lgr.error)(e_str)\n\n                if \"Clone succeeded, but checkout failed.\" in str(e):\n                    fix_annex = e\n                    break\n\n                raise\n\n        # get ourselves a repository instance\n        gr = cls(path, *args, **kwargs)\n        if fix_annex:\n            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then\n            # would need to import our own subclass\n            if hasattr(gr, 'is_valid_annex'):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                if not gr.is_initialized():\n                    gr._init()\n                gr.fsck()\n            else:\n                lgr.warning(\"Experienced issues while cloning: %s\", exc_str(fix_annex))\n        return gr\n\n    def __del__(self):\n        # unbind possibly bound ConfigManager, to prevent all kinds of weird\n        # stalls etc\n        self._cfg = None\n        # Make sure to flush pending changes, especially close batch processes\n        # (internal `git cat-file --batch` by GitPython)\n        try:\n            if getattr(self, '_repo', None) is not None and exists(self.path):\n                # gc might be late, so the (temporary)\n                # repo doesn't exist on FS anymore\n                self._repo.git.clear_cache()\n                # We used to write out the index to flush GitPython's\n                # state... but such unconditional write is really a workaround\n                # and does not play nice with read-only operations - permission\n                # denied etc. So disabled \n                #if exists(opj(self.path, '.git')):  # don't try to write otherwise\n                #    self.repo.index.write()\n        except (InvalidGitRepositoryError, AttributeError):\n            # might have being removed and no longer valid or attributes unbound\n            pass\n\n    def __repr__(self):\n        return \"<GitRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def __eq__(self, obj):\n        \"\"\"Decides whether or not two instances of this class are equal.\n\n        This is done by comparing the base repository path.\n        \"\"\"\n        return self.path == obj.path\n\n    def is_valid_git(self):\n        \"\"\"Returns whether the underlying repository appears to be still valid\n\n        Note, that this almost identical to the classmethod is_valid_repo(). However,\n        if we are testing an existing instance, we can save Path object creations. Since this testing\n        is done a lot, this is relevant. Creation of the Path objects in is_valid_repo() takes nearly half the time of\n        the entire function.\n\n        Also note, that this method is bound to an instance but still class-dependent, meaning that a subclass\n        cannot simply overwrite it. This is particularly important for the call from within __init__(),\n        which in turn is called by the subclasses' __init__. Using an overwrite would lead to the wrong thing being\n        called.\n        \"\"\"\n\n        return self.dot_git.exists() and (\n                not self.dot_git.is_dir() or self._valid_git_test_path.exists()\n        )\n\n    @classmethod\n    def is_valid_repo(cls, path):\n        \"\"\"Returns if a given path points to a git repository\"\"\"\n        if not isinstance(path, Path):\n            path = Path(path)\n        dot_git_path = path / '.git'\n\n        # the aim here is to have this test as cheap as possible, because\n        # it is performed a lot\n        # recognize two things as good-enough indicators of a present\n        # repo: 1) a non-empty .git directory (#3473)\n        #          NOTE: It's actually faster (and more accurate) to test for existence of a particular subpath.\n        #                This should be something that's there right after git-init. Going for .git/HEAD ATM.\n        #\n        #                In [11]: %timeit path.exists() and (not path.is_dir() or head_path.exists())\n        #                4.93 µs ± 34.8 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n        #                In [12]: %timeit path.exists() and (not path.is_dir() or any(path.iterdir()))\n        #                12.8 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n        #\n        #   and 2) a pointer file or symlink\n\n        return dot_git_path.exists() and (\n            not dot_git_path.is_dir() or (dot_git_path / 'HEAD').exists()\n        )\n\n    @staticmethod\n    def get_git_dir(repo):\n        \"\"\"figure out a repo's gitdir\n\n        '.git' might be a  directory, a symlink or a file\n\n        Note\n        ----\n        Please try using GitRepo.dot_git instead! That one's not static, but it's cheaper and you should avoid\n        not having an instance of a repo you're working on anyway. Note, that the property in opposition to this method\n        returns an absolute path.\n\n\n        Parameters\n        ----------\n        repo: path or Repo instance\n          currently expected to be the repos base dir\n\n        Returns\n        -------\n        str\n          relative path to the repo's git dir; So, default would be \".git\"\n        \"\"\"\n        if hasattr(repo, 'path'):\n            # repo instance like given\n            repo = repo.path\n        dot_git = op.join(repo, \".git\")\n        if not op.exists(dot_git):\n            raise RuntimeError(\"Missing .git in %s.\" % repo)\n        elif op.islink(dot_git):\n            git_dir = os.readlink(dot_git)\n        elif op.isdir(dot_git):\n            git_dir = \".git\"\n        elif op.isfile(dot_git):\n            with open(dot_git) as f:\n                git_dir = f.readline()\n                if git_dir.startswith(\"gitdir:\"):\n                    git_dir = git_dir[7:]\n                git_dir = git_dir.strip()\n\n        return git_dir\n\n    @property\n    def config(self):\n        \"\"\"Get an instance of the parser for the persistent repository\n        configuration.\n\n        Note: This allows to also read/write .datalad/config,\n        not just .git/config\n\n        Returns\n        -------\n        ConfigManager\n        \"\"\"\n        if self._cfg is None:\n            # associate with this dataset and read the entire config hierarchy\n            self._cfg = ConfigManager(dataset=self, source='any')\n        return self._cfg\n\n    def is_with_annex(self):\n        \"\"\"Report if GitRepo (assumed) has (remotes with) a git-annex branch\n        \"\"\"\n        return any(\n            b['refname:strip=2'] == 'git-annex' or b['refname:strip=2'].endswith('/git-annex')\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern=['refs/heads', 'refs/remotes'])\n        )\n\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n        cmd = ['git']\n        if git_options:\n            cmd.extend(git_options)\n        cmd += [\"rev-parse\", \"--show-toplevel\"]\n        try:\n            toppath, err = GitRunner().run(\n                cmd,\n                cwd=path,\n                log_stdout=True, log_stderr=True,\n                expect_fail=True, expect_stderr=True)\n            toppath = toppath.rstrip('\\n\\r')\n        except CommandError:\n            return None\n        except OSError:\n            toppath = GitRepo.get_toppath(dirname(path), follow_up=follow_up,\n                                          git_options=git_options)\n\n        if follow_up:\n            path_ = path\n            path_prev = \"\"\n            while path_ and path_ != path_prev:  # on top /.. = /\n                if realpath(path_) == toppath:\n                    toppath = path_\n                    break\n                path_prev = path_\n                path_ = dirname(path_)\n\n        return toppath\n\n    @normalize_paths\n    def add(self, files, git=True, git_options=None, update=False):\n        \"\"\"Adds file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list\n          list of paths to add\n        git: bool\n          somewhat ugly construction to be compatible with AnnexRepo.add();\n          has to be always true.\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n        Returns\n        -------\n        list\n          Of status dicts.\n        \"\"\"\n        # under all circumstances call this class' add_ (otherwise\n        # AnnexRepo.add would go into a loop\n        return list(GitRepo.add_(self, files, git=git, git_options=git_options,\n                    update=update))\n\n    def add_(self, files, git=True, git_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        # TODO: git_options is used as options for the git-add here,\n        # instead of options to the git executable => rename for consistency\n\n        if not git:\n            lgr.warning(\n                'GitRepo.add() called with git=%s, this should not happen',\n                git)\n            git = True\n\n        # there is no other way then to collect all files into a list\n        # at this point, because we need to pass them at once to a single\n        # `git add` call\n        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]\n\n        if not (files or git_options or update):\n            # wondering why just a warning? in cmdline this is also not an error\n            lgr.warning(\"add was called with empty file list and no options.\")\n            return\n\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                files,\n                # Set annex.largefiles to prevent storing files in annex when\n                # GitRepo() is instantiated with a v6+ annex repo.\n                ['git', '-c', 'annex.largefiles=nothing', 'add'] +\n                ensure_list(git_options) +\n                to_options(update=update) + ['--verbose']\n            )\n            # get all the entries\n            for o in self._process_git_get_output(*add_out):\n                yield o\n            # Note: as opposed to git cmdline, force is True by default in\n            #       gitpython, which would lead to add things, that are\n            #       ignored or excluded otherwise\n            # 2. Note: There is an issue with globbing (like adding '.'),\n            #       which apparently doesn't care for 'force' and therefore\n            #       adds '.git/...'. May be it's expanded at the wrong\n            #       point in time or sth. like that.\n            # For now, use direct call to git add.\n            #self.cmd_call_wrapper(self.repo.index.add, files, write=True,\n            #                      force=False)\n            # TODO: May be make use of 'fprogress'-option to indicate\n            # progress\n            # But then, we don't have it for git-annex add, anyway.\n            #\n            # TODO: Is write=True a reasonable way to do it?\n            # May be should not write until success of operation is\n            # confirmed?\n            # What's best in case of a list of files?\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n        # Make sure return value from GitRepo is consistent with AnnexRepo\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return\n\n    @staticmethod\n    def _process_git_get_output(stdout, stderr=None):\n        \"\"\"Given both outputs (stderr is ignored atm) of git add - process it\n\n        Primarily to centralize handling in both indirect annex and direct\n        modes when ran through proxy\n        \"\"\"\n        return [{u'file': f, u'success': True}\n                for f in re.findall(\"'(.*)'[\\n$]\", ensure_unicode(stdout))]\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, recursive=False, **kwargs):\n        \"\"\"Remove files.\n\n        Calls git-rm.\n\n        Parameters\n        ----------\n        files: str\n          list of paths to remove\n        recursive: False\n          whether to allow recursive removal from subdirectories\n        kwargs:\n          see `__init__`\n\n        Returns\n        -------\n        [str]\n          list of successfully removed files.\n        \"\"\"\n\n        files = _remove_empty_items(files)\n\n        if recursive:\n            kwargs['r'] = True\n        stdout, stderr = self._git_custom_command(\n            files, ['git', 'rm'] + to_options(**kwargs))\n\n        # output per removed file is expected to be \"rm 'PATH'\":\n        return [line.strip()[4:-1] for line in stdout.splitlines()]\n\n        #return self.repo.git.rm(files, cached=False, **kwargs)\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks\n        \"\"\"\n        # All GitPython commands should take care about flushing index\n        # whenever they modify it, so we would not care to do anything\n        # if self.repo is not None and exists(opj(self.path, '.git')):  # don't try to write otherwise:\n        #     # flush possibly cached in GitPython changes to index:\n        #     # if self.repo.git:\n        #     #     sys.stderr.write(\"CLEARING\\n\")\n        #     #     self.repo.git.clear_cache()\n        #     self.repo.index.write()\n\n        # Close batched by GitPython git processes etc\n        # Ref: https://github.com/gitpython-developers/GitPython/issues/718\n        self.repo.__del__()\n        pass\n\n    @staticmethod\n    def _get_prefixed_commit_msg(msg):\n        DATALAD_PREFIX = \"[DATALAD]\"\n        return DATALAD_PREFIX if not msg else \"%s %s\" % (DATALAD_PREFIX, msg)\n\n    def for_each_ref_(self, fields=('objectname', 'objecttype', 'refname'),\n                      pattern=None, points_at=None, sort=None, count=None,\n                      contains=None):\n        \"\"\"Wrapper for `git for-each-ref`\n\n        Please see manual page git-for-each-ref(1) for a complete overview\n        of its functionality. Only a subset of it is supported by this\n        wrapper.\n\n        Parameters\n        ----------\n        fields : iterable or str\n          Used to compose a NULL-delimited specification for for-each-ref's\n          --format option. The default field list reflects the standard\n          behavior of for-each-ref when the --format option is not given.\n        pattern : list or str, optional\n          If provided, report only refs that match at least one of the given\n          patterns.\n        points_at : str, optional\n          Only list refs which points at the given object.\n        sort : list or str, optional\n          Field name(s) to sort-by. If multiple fields are given, the last one\n          becomes the primary key. Prefix any field name with '-' to sort in\n          descending order.\n        count : int, optional\n          Stop iteration after the given number of matches.\n        contains : str, optional\n          Only list refs which contain the specified commit.\n\n        Yields\n        ------\n        dict with items matching the given `fields`\n\n        Raises\n        ------\n        ValueError\n          if no `fields` are given\n\n        RuntimeError\n          if `git for-each-ref` returns a record where the number of\n          properties does not match the number of `fields`\n        \"\"\"\n        if not fields:\n            raise ValueError('no `fields` provided, refuse to proceed')\n        fields = ensure_list(fields)\n        cmd = [\n            \"git\",\n            \"for-each-ref\",\n            \"--format={}\".format(\n                '%00'.join(\n                    '%({})'.format(f) for f in fields)),\n        ]\n        if points_at:\n            cmd.append('--points-at={}'.format(points_at))\n        if contains:\n            cmd.append('--contains={}'.format(contains))\n        if sort:\n            for k in ensure_list(sort):\n                cmd.append('--sort={}'.format(k))\n        if pattern:\n            cmd += ensure_list(pattern)\n        if count:\n            cmd.append('--count={:d}'.format(count))\n\n        out, _ = self._git_custom_command(None, cmd)\n        for line in out.splitlines():\n            props = line.split('\\0')\n            if len(fields) != len(props):\n                raise RuntimeError(\n                    'expected fields {} from git-for-each-ref, but got: {}'.format(\n                        fields, props))\n            yield dict(zip(fields, props))\n\n    def configure_fake_dates(self):\n        \"\"\"Configure repository to use fake dates.\n        \"\"\"\n        lgr.debug(\"Enabling fake dates\")\n        self.config.set(\"datalad.fake-dates\", \"true\")\n\n    @property\n    def fake_dates_enabled(self):\n        \"\"\"Is the repository configured to use fake dates?\n        \"\"\"\n        if self._fake_dates_enabled is None:\n            self._fake_dates_enabled = \\\n                self.config.getbool('datalad', 'fake-dates', default=False)\n        return self._fake_dates_enabled\n\n    def add_fake_dates(self, env):\n        \"\"\"Add fake dates to `env`.\n\n        Parameters\n        ----------\n        env : dict or None\n            Environment variables.\n\n        Returns\n        -------\n        A dict (copied from env), with date-related environment\n        variables for git and git-annex set.\n        \"\"\"\n        env = (env if env is not None else os.environ).copy()\n        # Note: Use _git_custom_command here rather than repo.git.for_each_ref\n        # so that we use annex-proxy in direct mode.\n        last_date = list(self.for_each_ref_(\n            fields='committerdate:raw',\n            count=1,\n            pattern='refs/heads',\n            sort=\"-committerdate\",\n        ))\n\n        if last_date:\n            # Drop the \"contextual\" timezone, leaving the unix timestamp.  We\n            # avoid :unix above because it wasn't introduced until Git v2.9.4.\n            last_date = last_date[0]['committerdate:raw'].split()[0]\n            seconds = int(last_date)\n        else:\n            seconds = self.config.obtain(\"datalad.fake-dates-start\")\n        seconds_new = seconds + 1\n        date = \"@{} +0000\".format(seconds_new)\n\n        lgr.debug(\"Setting date to %s\",\n                  time.strftime(\"%a %d %b %Y %H:%M:%S +0000\",\n                                time.gmtime(seconds_new)))\n\n        env[\"GIT_AUTHOR_DATE\"] = date\n        env[\"GIT_COMMITTER_DATE\"] = date\n        env[\"GIT_ANNEX_VECTOR_CLOCK\"] = str(seconds_new)\n\n        return env\n\n    def commit(self, msg=None, options=None, _datalad_msg=False, careless=True,\n               files=None, date=None, index_file=None):\n        \"\"\"Commit changes to git.\n\n        Parameters\n        ----------\n        msg: str, optional\n          commit-message\n        options: list of str, optional\n          cmdline options for git-commit\n        _datalad_msg: bool, optional\n          To signal that commit is automated commit by datalad, so\n          it would carry the [DATALAD] prefix\n        careless: bool, optional\n          if False, raise when there's nothing actually committed;\n          if True, don't care\n        files: list of str, optional\n          path(s) to commit\n        date: str, optional\n          Date in one of the formats git understands\n        index_file: str, optional\n          An alternative index to use\n        \"\"\"\n\n        self.precommit()\n\n        # assemble commandline\n        cmd = ['git', 'commit']\n        options = ensure_list(options)\n\n        if date:\n            options += [\"--date\", date]\n\n        if not msg:\n            msg = 'Recorded changes'\n            _datalad_msg = True\n\n        if _datalad_msg:\n            msg = self._get_prefixed_commit_msg(msg)\n\n        options += [\"-m\", msg]\n        cmd.extend(options)\n\n        # set up env for commit\n        env = GitRunner.get_git_environ_adjusted()\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n        if index_file:\n            env['GIT_INDEX_FILE'] = index_file\n\n        lgr.debug(\"Committing via direct call of git: %s\" % cmd)\n\n        file_chunks = generate_file_chunks(files, cmd) if files else [[]]\n\n        runner = WitlessRunner(cwd=self.path, env=env)\n\n        try:\n            for i, chunk in enumerate(file_chunks):\n                cur_cmd = cmd + (\n                    # if this is an explicit dry-run, there is no point in\n                    # amending, because no commit was ever made\n                    # otherwise, amend the first commit, and prevent\n                    # leaving multiple commits behind\n                    ['--amend', '--no-edit']\n                    if i > 0 and '--dry-run' not in cmd\n                    else []\n                ) + ['--'] + chunk\n                runner.run(\n                    cur_cmd,\n                    protocol=StdOutErrCapture,\n                    stdin=None,\n                )\n        except CommandError as e:\n            # real errors first\n            if \"did not match any file(s) known to git\" in e.stderr:\n                raise FileNotInRepositoryError(\n                    cmd=e.cmd,\n                    msg=\"File(s) unknown to git\",\n                    code=e.code,\n                    filename=linesep.join([\n                        l for l in e.stderr.splitlines()\n                        if l.startswith(\"error: pathspec\")\n                    ])\n                )\n            # behavior choices now\n            elif not careless:\n                # not willing to compromise at all\n                raise\n            elif 'nothing to commit' in e.stdout:\n                lgr.debug(\"nothing to commit in %s. Ignored.\", self)\n            elif 'no changes added to commit' in e.stdout or \\\n                    'nothing added to commit' in e.stdout:\n                lgr.debug(\"no changes added to commit in %s. Ignored.\", self)\n            else:\n                raise\n\n    # TODO usage is primarily in the tests, consider making a test helper and\n    # remove from GitRepo API\n    def get_indexed_files(self):\n        \"\"\"Get a list of files in git's index\n\n        Returns\n        -------\n        list\n            list of paths rooting in git's base dir\n        \"\"\"\n\n        return [\n            str(r.relative_to(self.pathobj))\n            for r in self.get_content_info(\n                paths=None, ref=None, untracked='no', eval_file_type=False)\n        ]\n\n    def format_commit(self, fmt, commitish=None):\n        \"\"\"Return `git show` output for `commitish`.\n\n        Parameters\n        ----------\n        fmt : str\n            A format string accepted by `git show`.\n        commitish: str, optional\n          Any commit identifier (defaults to \"HEAD\").\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        cmd = ['git', 'show', '-z', '--no-patch', '--format=' + fmt]\n        if commitish is not None:\n            cmd.append(commitish + \"^{commit}\")\n        # make sure Git takes our argument as a revision\n        cmd.append('--')\n        try:\n            stdout, stderr = self._git_custom_command(\n                '', cmd, expect_stderr=True, expect_fail=True)\n        except CommandError as e:\n            if 'bad revision' in e.stderr:\n                raise ValueError(\"Unknown commit identifier: %s\" % commitish)\n            elif 'does not have any commits yet' in e.stderr:\n                return None\n            else:\n                raise e\n        # This trailing null is coming from the -z above, which avoids the\n        # newline that Git would append to the output. We could drop -z and\n        # strip the newline directly, but then we'd have to worry about\n        # compatibility across platforms.\n        return stdout.rsplit(\"\\0\", 1)[0]\n\n    def get_hexsha(self, commitish=None, short=False):\n        \"\"\"Return a hexsha for a given commitish.\n\n        Parameters\n        ----------\n        commitish : str, optional\n          Any identifier that refers to a commit (defaults to \"HEAD\").\n        short : bool, optional\n          Return the abbreviated form of the hexsha.\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        stdout = self.format_commit(\"%{}\".format('h' if short else 'H'),\n                                    commitish)\n        if stdout is not None:\n            stdout = stdout.splitlines()\n            assert(len(stdout) == 1)\n            return stdout[0]\n\n    @normalize_paths(match_return_type=False)\n    def get_last_commit_hexsha(self, files):\n        \"\"\"Return the hash of the last commit the modified any of the given\n        paths\"\"\"\n        try:\n            commit = self.call_git(\n                ['rev-list', '-n1', 'HEAD'],\n                files=files,\n                expect_fail=True,\n            )\n            commit = commit.strip()\n            return commit if commit else None\n        except CommandError:\n            if self.get_hexsha() is None:\n                # unborn branch, don't freak out\n                return None\n            raise\n\n    def get_revisions(self, revrange=None, fmt=\"%H\", options=None):\n        \"\"\"Return list of revisions in `revrange`.\n\n        Parameters\n        ----------\n        revrange : str or list of str or None, optional\n            Revisions or revision ranges to walk. If None, revision defaults to\n            HEAD unless a revision-modifying option like `--all` or\n            `--branches` is included in `options`.\n        fmt : string, optional\n            Format accepted by `--format` option of `git log`. This should not\n            contain new lines because the output is split on new lines.\n        options : list of str, optional\n            Options to pass to `git log`.  This should not include `--format`.\n\n        Returns\n        -------\n        List of revisions (str), formatted according to `fmt`.\n        \"\"\"\n        if revrange is None:\n            revrange = []\n        elif isinstance(revrange, str):\n            revrange = [revrange]\n\n        cmd = [\"git\", \"log\", \"--format={}\".format(fmt)]\n        cmd.extend((options or []) + revrange + [\"--\"])\n        try:\n            stdout, _ = self._git_custom_command(None, cmd, expect_fail=True)\n        except CommandError as e:\n            if \"does not have any commits\" in e.stderr:\n                return []\n            raise\n        return stdout.splitlines()\n\n    def commit_exists(self, commitish):\n        \"\"\"Does `commitish` exist in the repo?\n\n        Parameters\n        ----------\n        commitish : str\n            A commit or an object that can be dereferenced to one.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        # Note: The peeling operator \"^{commit}\" is required so that rev-parse\n        # doesn't succeed if passed a full hexsha that is valid but doesn't\n        # exist.\n        return self.call_git_success(\n            [\"rev-parse\", \"--verify\", commitish + \"^{commit}\"])\n\n    def get_merge_base(self, commitishes):\n        \"\"\"Get a merge base hexsha\n\n        Parameters\n        ----------\n        commitishes: str or list of str\n          List of commitishes (branches, hexshas, etc) to determine the merge\n          base of. If a single value provided, returns merge_base with the\n          current branch.\n\n        Returns\n        -------\n        str or None\n          If no merge-base for given commits, or specified treeish doesn't\n          exist, None returned\n        \"\"\"\n        if isinstance(commitishes, str):\n            commitishes = [commitishes]\n        if not commitishes:\n            raise ValueError(\"Provide at least a single value\")\n        elif len(commitishes) == 1:\n            commitishes = commitishes + [self.get_active_branch()]\n\n        try:\n            base = self.call_git_oneline(['merge-base'] + commitishes)\n        except CommandError as exc:\n            if exc.code == 1 and not (exc.stdout or exc.stderr):\n                # No merge base was found (unrelated commits).\n                return None\n            if \"fatal: Not a valid object name\" in exc.stderr:\n                return None\n            raise\n\n        return base\n\n    def is_ancestor(self, reva, revb):\n        \"\"\"Is `reva` an ancestor of `revb`?\n\n        Parameters\n        ----------\n        reva, revb : str\n            Revisions.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return self.call_git_success(\n            [\"merge-base\", \"--is-ancestor\", reva, revb])\n\n    def get_commit_date(self, branch=None, date='authored'):\n        \"\"\"Get the date stamp of the last commit (in a branch or head otherwise)\n\n        Parameters\n        ----------\n        date: {'authored', 'committed'}\n          Which date to return.  \"authored\" will be the date shown by \"git show\"\n          and the one possibly specified via --date to `git commit`\n\n        Returns\n        -------\n        int or None\n          None if no commit\n        \"\"\"\n        if date == 'committed':\n            format = '%ct'\n        elif date == 'authored':\n            format = '%at'\n        else:\n            raise ValueError('unknow date type: {}'.format(date))\n        d = self.format_commit(format, commitish=branch)\n        return int(d) if d else None\n\n    def get_active_branch(self):\n        \"\"\"Get the name of the active branch\n\n        Returns\n        -------\n        str or None\n          Returns None if there is no active branch, i.e. detached HEAD,\n          and the branch name otherwise.\n        \"\"\"\n        try:\n            out, _ = self._git_custom_command(\n                \"\", [\"git\", \"symbolic-ref\", \"HEAD\"],\n                expect_fail=True)\n        except CommandError as e:\n            if 'HEAD is not a symbolic ref' in e.stderr:\n                lgr.debug(\"detached HEAD in {0}\".format(self))\n                return None\n            else:\n                raise e\n        return out.strip()[11:]  # strip refs/heads/\n\n    def get_branches(self):\n        \"\"\"Get all branches of the repo.\n\n        Returns\n        -------\n        [str]\n            Names of all branches of this repository.\n        \"\"\"\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/heads')\n        ]\n\n    def get_remote_branches(self):\n        \"\"\"Get all branches of all remotes of the repo.\n\n        Returns\n        -----------\n        [str]\n            Names of all remote branches.\n        \"\"\"\n        # TODO: Reconsider melting with get_branches()\n\n        # TODO: treat entries like this: origin/HEAD -> origin/master'\n        # currently this is done in collection\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/remotes')\n        ]\n\n    def get_remotes(self, with_urls_only=False):\n        \"\"\"Get known remotes of the repository\n\n        Parameters\n        ----------\n        with_urls_only : bool, optional\n          return only remotes which have urls\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n\n        # Note: read directly from config and spare instantiation of gitpy.Repo\n        # since we need this in AnnexRepo constructor. Furthermore gitpy does it\n        # pretty much the same way and the use of a Repo instance seems to have\n        # no reason other than a nice object oriented look.\n        from datalad.utils import unique\n\n        self.config.reload()\n        remotes = unique([x[7:] for x in self.config.sections()\n                          if x.startswith(\"remote.\")])\n\n        if with_urls_only:\n            remotes = [\n                r for r in remotes\n                if self.config.get('remote.%s.url' % r)\n            ]\n        return remotes\n\n    # TODO this is practically unused outside the tests, consider turning\n    # into a test helper and trim from the API\n    def get_files(self, branch=None):\n        \"\"\"Get a list of files in git.\n\n        Lists the files in the (remote) branch.\n\n        Parameters\n        ----------\n        branch: str\n          Name of the branch to query. Default: active branch.\n\n        Returns\n        -------\n        [str]\n          list of files.\n        \"\"\"\n        return [\n            str(p.relative_to(self.pathobj))\n            for p in self.get_content_info(\n                paths=None, ref=branch, untracked='no', eval_file_type=False)\n            ]\n\n    @normalize_paths(match_return_type=False)\n    def _git_custom_command(self, files, cmd_str,\n                            log_stdout=True, log_stderr=True, log_online=False,\n                            expect_stderr=True, cwd=None, env=None,\n                            shell=None, expect_fail=False,\n                            check_fake_dates=False,\n                            index_file=None,\n                            updates_tree=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git commands\n        for proof of concept without the need to figure out, how this is done\n        via GitPython.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str or list\n          arbitrary command str. `files` is appended to that string.\n        updates_tree: bool\n          whether or not command updates the working tree. If True, triggers\n          necessary reevaluations like self.config.reload()\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n\n        # ensure cmd_str becomes a well-formed list:\n        if isinstance(cmd_str, str):\n            cmd = split_cmdline(cmd_str)\n        else:\n            cmd = cmd_str[:]  # we will modify in-place\n\n        assert(cmd[0] == 'git')\n        cmd = cmd[:1] + self._GIT_COMMON_OPTIONS + cmd[1:]\n\n        if check_fake_dates and self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if index_file:\n            env = (env if env is not None else os.environ).copy()\n            env['GIT_INDEX_FILE'] = index_file\n\n        # TODO?: wouldn't splitting interfer with above GIT_INDEX_FILE\n        #  handling????\n        try:\n            out, err = run_gitcommand_on_file_list_chunks(\n                self.cmd_call_wrapper.run,\n                cmd,\n                files,\n                log_stderr=log_stderr,\n                log_stdout=log_stdout,\n                log_online=log_online,\n                expect_stderr=expect_stderr,\n                cwd=cwd,\n                env=env,\n                shell=shell,\n                expect_fail=expect_fail)\n        except CommandError as e:\n            ignored = re.search(GitIgnoreError.pattern, e.stderr)\n            if ignored:\n                raise GitIgnoreError(cmd=e.cmd, msg=e.stderr,\n                                     code=e.code, stdout=e.stdout,\n                                     stderr=e.stderr,\n                                     paths=ignored.groups()[0].splitlines())\n            raise\n\n        if updates_tree:\n            lgr.debug(\"Reloading config due to supposed working tree update\")\n            self.config.reload()\n\n        return out, err\n\n    # Convenience wrappers for one-off git calls that don't require further\n    # processing or error handling.\n\n    def call_git(self, args, files=None,\n                 expect_stderr=False, expect_fail=False):\n        \"\"\"Call git and return standard output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        expect_fail : bool, optional\n          A non-zero exit is expected and should not be elevated above the\n          DEBUG level.\n\n        Returns\n        -------\n        standard output (str)\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          expect_fail=expect_fail,\n                                          check_fake_dates=True)\n        return out\n\n    def call_git_items_(self, args, files=None, expect_stderr=False, sep=None):\n        \"\"\"Call git, splitting output on `sep`.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Returns\n        -------\n        Generator that yields output items.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          check_fake_dates=True)\n        yield from (out.split(sep) if sep else out.splitlines())\n\n    def call_git_oneline(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git for a single line of output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        AssertionError if there is more than one line of output.\n        \"\"\"\n        lines = list(self.call_git_items_(args, files=files,\n                                          expect_stderr=expect_stderr))\n        if len(lines) > 1:\n            raise AssertionError(\n                \"Expected {} to return single line, but it returned {}\"\n                .format([\"git\"] + args, lines))\n        return lines[0]\n\n    def call_git_success(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git and return true if the call exit code of 0.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        try:\n            self._git_custom_command(files, [\"git\"] + args,\n                                     expect_fail=True,\n                                     expect_stderr=expect_stderr,\n                                     check_fake_dates=True)\n        except CommandError:\n            return False\n        return True\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Register remote pointing to a url\n        \"\"\"\n        cmd = ['git', 'remote', 'add']\n        if options:\n            cmd += options\n        cmd += [name, url]\n\n        result = self._git_custom_command('', cmd)\n        self.config.reload()\n        return result\n\n    def remove_remote(self, name):\n        \"\"\"Remove existing remote\n        \"\"\"\n\n        # TODO: testing and error handling!\n        from .exceptions import RemoteNotAvailableError\n        try:\n            out, err = self._git_custom_command(\n                '', ['git', 'remote', 'remove', name])\n        except CommandError as e:\n            if 'fatal: No such remote' in e.stderr:\n                raise RemoteNotAvailableError(name,\n                                              cmd=\"git remote remove\",\n                                              msg=\"No such remote\",\n                                              stdout=e.stdout,\n                                              stderr=e.stderr)\n            else:\n                raise e\n\n        # TODO: config.reload necessary?\n        self.config.reload()\n        return\n\n    def update_remote(self, name=None, verbose=False):\n        \"\"\"\n        \"\"\"\n        options = [\"-v\"] if verbose else []\n        name = [name] if name else []\n        self._git_custom_command(\n            '', ['git', 'remote'] + name + ['update'] + options,\n            expect_stderr=True\n        )\n\n    def fetch(self, remote=None, refspec=None, all_=False, git_options=None,\n              **kwargs):\n        \"\"\"Fetches changes from a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to fetch from. If no remote is given and\n          `all_` is not set, the tracking branch is fetched.\n        refspec : str, optional\n          refspec to fetch.\n        all_ : bool, optional\n          fetch all remotes (and all of their branches).\n          Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-fetch.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-fetch.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        return list(\n            self.fetch_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    @guard_BadName\n    def fetch_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `fetch`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'fetch', '--verbose', '--progress'],\n            action='fetch',\n            urlvars=('remote.{}.url', 'remote.{}.url'),\n            protocol=GitProgress,\n            info_cls=FetchInfo,\n            info_from=1,\n            add_remote=False,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    # XXX Consider removing this method. It is only used in `update()`,\n    # where it could be easily replaced with fetch+merge\n    def pull(self, remote=None, refspec=None, git_options=None, **kwargs):\n        \"\"\"Pulls changes from a remote.\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to pull from. If no remote is given,\n          the remote tracking branch is used.\n        refspec : str, optional\n          refspec to fetch.\n        git_options : list, optional\n          Additional command line options for git-pull.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-pull.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n\n        cmd = ['git', 'pull', '--progress'] + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or fetch all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            # No explicit remote to fetch.\n            # => get tracking branch:\n            tb_remote, refspec = self.get_tracking_branch()\n            if tb_remote is not None:\n                remote = tb_remote\n            else:\n                # No remote, no tracking branch\n                # => fail\n                raise ValueError(\"Neither a remote is specified to pull \"\n                                 \"from nor a tracking branch is set up.\")\n\n        cmd.append(remote)\n        if refspec:\n            cmd += ensure_list(refspec)\n\n        # best effort to enable SSH connection caching\n        url = self.config.get('remote.{}.url'.format(remote), None)\n        if url and is_ssh(url):\n            ssh_manager.get_connection(url).open()\n        WitlessRunner(\n            cwd=self.path,\n            env=GitRunner.get_git_environ_adjusted()).run(\n                cmd,\n                protocol=StdOutCaptureWithGitProgress,\n        )\n\n    def push(self, remote=None, refspec=None, all_remotes=False,\n             all_=False, git_options=None, **kwargs):\n        \"\"\"Push changes to a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to push to. If no remote is given and\n          `all_` is not set, the tracking branch is pushed.\n        refspec : str, optional\n          refspec to push.\n        all_ : bool, optional\n          push to all remotes. Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-push.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-push.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        if all_remotes:\n            # be nice to the elderly\n            all_ = True\n        return list(\n            self.push_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    def push_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `push`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'push', '--progress', '--porcelain'],\n            action='push',\n            urlvars=('remote.{}.pushurl', 'remote.{}.url'),\n            protocol=StdOutCaptureWithGitProgress,\n            info_cls=PushInfo,\n            info_from=0,\n            add_remote=True,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    def _fetch_push_helper(\n            self,\n            base_cmd,     # arg list\n            action,       # label fetch|push\n            urlvars,      # variables to query for URLs\n            protocol,     # processor for output\n            info_cls,     # Push|FetchInfo\n            info_from,    # 0=stdout, 1=stderr\n            add_remote,   # whether to add a 'remote' field to the info dict\n            remote=None, refspec=None, all_=False, git_options=None):\n\n        git_options = ensure_list(git_options)\n\n        cmd = base_cmd + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or push all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            if all_:\n                remotes_to_process = self.get_remotes(with_urls_only=True)\n            else:\n                # No explicit remote to fetch.\n                # => get tracking branch:\n                tb_remote, refspec = self.get_tracking_branch()\n                if tb_remote is not None:\n                    remotes_to_process = [tb_remote]\n                else:\n                    # No remote, no tracking branch\n                    # => fail\n                    raise ValueError(\n                        \"Neither a remote is specified to {} \"\n                        \"from nor a tracking branch is set up.\".format(action))\n        else:\n            if all_:\n                raise ValueError(\n                    \"Option 'all_' conflicts with specified remote \"\n                    \"'{}'.\".format(remote))\n            remotes_to_process = [remote]\n\n        if refspec:\n            # prep for appending to cmd\n            refspec = ensure_list(refspec)\n\n        # no need for progress report, when there is just one remote\n        log_remote_progress = len(remotes_to_process) > 1\n        if log_remote_progress:\n            pbar_id = '{}remotes-{}'.format(action, id(self))\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start %sing remotes for %s', action, self,\n                total=len(remotes_to_process),\n                label=action.capitalize(),\n                unit=' Remotes',\n            )\n        try:\n            for remote in remotes_to_process:\n                r_cmd = cmd + [remote]\n                if refspec:\n                    r_cmd += refspec\n\n                if log_remote_progress:\n                    log_progress(\n                        lgr.info,\n                        pbar_id,\n                        '{}ing remote %s'.format(action.capitalize()),\n                        remote,\n                        update=1,\n                        increment=True,\n                    )\n                # best effort to enable SSH connection caching\n                url = self.config.get(\n                    # make two attempts to get a URL\n                    urlvars[0].format(remote),\n                    self.config.get(\n                        urlvars[1].format(remote),\n                        None)\n                )\n                if url and is_ssh(url):\n                    ssh_manager.get_connection(url).open()\n                try:\n                    out = WitlessRunner(\n                        cwd=self.path,\n                        env=GitRunner.get_git_environ_adjusted()).run(\n                            r_cmd,\n                            protocol=protocol,\n                    )\n                    output = out[info_from] or ''\n                except CommandError as e:\n                    # intercept some errors that we express as an error report\n                    # in the info dicts\n                    if re.match('error: failed to (push|fetch) some refs', e.stderr):\n                        output = {1: e.stderr, 0: e.stdout}[info_from]\n                        if output is None:\n                            output = ''\n                    else:\n                        raise\n\n                for line in output.splitlines():\n                    try:\n                        # push info doesn't identify a remote, add it here\n                        pi = info_cls._from_line(line)\n                        if add_remote:\n                            pi['remote'] = remote\n                        yield pi\n                    except Exception:\n                        # it is not progress and no push info\n                        # don't hide it completely\n                        lgr.debug('git-%s reported: %s', action, line)\n        finally:\n            if log_remote_progress:\n                log_progress(\n                    lgr.info,\n                    pbar_id,\n                    'Finished %sing remotes for %s', action, self,\n                )\n\n    def get_remote_url(self, name, push=False):\n        \"\"\"Get the url of a remote.\n\n        Reads the configuration of remote `name` and returns its url or None,\n        if there is no url configured.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        push: bool\n          if True, get the pushurl instead of the fetch url.\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        return self.config.get(var, None)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        self.config.set(var, url, where='local', reload=True)\n\n    def get_branch_commits_(self, branch=None, limit=None, stop=None):\n        \"\"\"Return commit hexshas for a branch\n\n        Parameters\n        ----------\n        branch: str, optional\n          If not provided, assumes current branch\n        limit: None | 'left-only', optional\n          Limit which commits to report.  If None -- all commits (merged or not),\n          if 'left-only' -- only the commits from the left side of the tree upon\n          merges\n        stop: str, optional\n          hexsha of the commit at which stop reporting (matched one is not\n          reported either)\n\n        Yields\n        ------\n        str\n        \"\"\"\n        cmd = ['rev-list']\n        if limit == 'left-only':\n            cmd.append('--left-only')\n        if not branch:\n            branch = self.get_active_branch()\n        cmd.append(branch)\n        for r in self.call_git_items_(cmd):\n            if stop and stop == r:\n                return\n            yield r\n\n    def checkout(self, name, options=None):\n        \"\"\"\n        \"\"\"\n        # TODO: May be check for the need of -b options herein?\n        cmd = ['git', 'checkout']\n        if options:\n            cmd += options\n        cmd += [str(name)]\n\n        self._git_custom_command('', cmd, expect_stderr=True, updates_tree=True)\n\n    # TODO: Before implementing annex merge, find usages and check for a needed\n    # change to call super().merge\n    def merge(self, name, options=None, msg=None, allow_unrelated=False, **kwargs):\n        if options is None:\n            options = []\n        if msg:\n            options = options + [\"-m\", msg]\n        if allow_unrelated and external_versions['cmd:git'] >= '2.9':\n            options += ['--allow-unrelated-histories']\n        self._git_custom_command(\n            '', ['git', 'merge'] + options + [name],\n            check_fake_dates=True,\n            **kwargs\n        )\n\n    def remove_branch(self, branch):\n        self._git_custom_command(\n            '', ['git', 'branch', '-D', branch]\n        )\n\n    def cherry_pick(self, commit):\n        \"\"\"Cherry pick `commit` to the current branch.\n\n        Parameters\n        ----------\n        commit : str\n            A single commit.\n        \"\"\"\n        self._git_custom_command(\"\", [\"git\", \"cherry-pick\", commit],\n                                 check_fake_dates=True)\n\n    @property\n    def dirty(self):\n        \"\"\"Is the repository dirty?\n\n        Note: This provides a quick answer when you simply want to know if\n        there are any untracked changes or modifications in this repository or\n        its submodules. For finer-grained control and more detailed reporting,\n        use status() instead.\n        \"\"\"\n        stdout, _ = self._git_custom_command(\n            [],\n            [\"git\", \"status\", \"--porcelain\",\n             # Ensure the result isn't influenced by status.showUntrackedFiles.\n             \"--untracked-files=normal\",\n             # Ensure the result isn't influenced by diff.ignoreSubmodules.\n             \"--ignore-submodules=none\"])\n        return bool(stdout.strip())\n\n    @property\n    def untracked_files(self):\n        \"\"\"Legacy interface, do not use! Use the status() method instead.\n\n        Despite its name, it also reports on untracked datasets, and\n        yields their names with trailing path separators.\n        \"\"\"\n        return [\n            '{}{}'.format(\n                str(p.relative_to(self.pathobj)),\n                os.sep if props['type'] != 'file' else ''\n            )\n            for p, props in self.status(\n                    untracked='all', eval_submodule_state='no').items()\n            if props.get('state', None) == 'untracked'\n        ]\n\n    def gc(self, allow_background=False, auto=False):\n        \"\"\"Perform house keeping (garbage collection, repacking)\"\"\"\n        cmd_options = ['git']\n        if not allow_background:\n            cmd_options += ['-c', 'gc.autodetach=0']\n        cmd_options += ['gc', '--aggressive']\n        if auto:\n            cmd_options += ['--auto']\n        self._git_custom_command('', cmd_options)\n\n    def _parse_gitmodules(self):\n        # TODO read .gitconfig from Git blob?\n        gitmodules = self.pathobj / '.gitmodules'\n        if not gitmodules.exists():\n            return {}\n        # pull out file content\n        out, err = self._git_custom_command(\n            '',\n            ['git', 'config', '-z', '-l', '--file', '.gitmodules'])\n        # abuse our config parser\n        db, _ = _parse_gitconfig_dump(out, {}, None, True, cwd=self.path)\n        mods = {}\n        for k, v in db.items():\n            if not k.startswith('submodule.'):\n                # we don't know what this is\n                lgr.warning(\"Skip unrecognized .gitmodule specification: %s=%s\", k, v)\n                continue\n            k_l = k.split('.')\n            # module name is everything after 'submodule.' that is not the variable\n            # name\n            mod_name = '.'.join(k_l[1:-1])\n            mod = mods.get(mod_name, {})\n            # variable name is the last 'dot-free' segment in the key\n            mod[k_l[-1]] = v\n            mods[mod_name] = mod\n\n        out = {}\n        # bring into traditional shape\n        for name, props in mods.items():\n            if 'path' not in props:\n                lgr.warning(\"Failed to get '%s.path', skipping this submodule\", name)\n                continue\n            modprops = {'gitmodule_{}'.format(k): v\n                        for k, v in props.items()\n                        if not (k.startswith('__') or k == 'path')}\n            modpath = self.pathobj / PurePosixPath(props['path'])\n            modprops['gitmodule_name'] = name\n            out[modpath] = modprops\n        return out\n\n    def get_submodules_(self, paths=None):\n        \"\"\"Yield submodules in this repository.\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n\n        Returns\n        -------\n        A generator that yields a dictionary with information for each\n        submodule.\n        \"\"\"\n        if not (self.pathobj / \".gitmodules\").exists():\n            return\n\n        modinfo = self._parse_gitmodules()\n        for path, props in self.get_content_info(\n                paths=paths,\n                ref=None,\n                untracked='no',\n                eval_file_type=False).items():\n            if props.get('type', None) != 'dataset':\n                continue\n            props[\"path\"] = path\n            props.update(modinfo.get(path, {}))\n            yield props\n\n    def get_submodules(self, sorted_=True, paths=None, compat=True):\n        \"\"\"Return list of submodules.\n\n        Parameters\n        ----------\n        sorted_ : bool, optional\n            Sort submodules by path name.\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n        compat : bool, optional\n            If true, return a namedtuple that incompletely mimics the\n            attributes of GitPython's Submodule object in hope of backwards\n            compatibility with previous callers. Note that this form should be\n            considered temporary and callers should be updated; this flag will\n            be removed in a future release.\n\n        Returns\n        -------\n        List of submodule namedtuples if `compat` is true or otherwise a list\n        of dictionaries as returned by `get_submodules_`.\n        \"\"\"\n        xs = self.get_submodules_(paths=paths)\n        if compat:\n            warnings.warn(\"The attribute-based return value of get_submodules() \"\n                          \"exists for compatibility purposes and will be removed \"\n                          \"in an upcoming release\",\n                          DeprecationWarning)\n            xs = (Submodule(name=p[\"gitmodule_name\"],\n                            path=str(p[\"path\"].relative_to(self.pathobj)),\n                            url=p[\"gitmodule_url\"])\n                  for p in xs)\n\n        if sorted_:\n            if compat:\n                def key(x):\n                    return x.path\n            else:\n                def key(x):\n                    return x[\"path\"]\n            xs = sorted(xs, key=key)\n        return list(xs)\n\n    def add_submodule(self, path, name=None, url=None, branch=None):\n        \"\"\"Add a new submodule to the repository.\n\n        This will alter the index as well as the .gitmodules file, but will not\n        create a new commit.  If the submodule already exists, no matter if the\n        configuration differs from the one provided, the existing submodule\n        is considered as already added and no further action is performed.\n\n        Parameters\n        ----------\n        path : str\n          repository-relative path at which the submodule should be located, and\n          which will be created as required during the repository initialization.\n        name : str or None\n          name/identifier for the submodule. If `None`, the `path` will be used\n          as name.\n        url : str or None\n          git-clone compatible URL. If `None`, the repository is assumed to\n          exist, and the url of the first remote is taken instead. This is\n          useful if you want to make an existing repository a submodule of\n          another one.\n        branch : str or None\n          name of branch to be checked out in the submodule. The given branch\n          must exist in the remote repository, and will be checked out locally\n          as a tracking branch. If `None`, remote HEAD will be checked out.\n        \"\"\"\n        if name is None:\n            name = Path(path).as_posix()\n        # XXX the following should do it, but GitPython will refuse to add a submodule\n        # unless you specify a URL that is configured as one of its remotes, or you\n        # specify no URL, but the repo has at least one remote.\n        # this is stupid, as for us it is valid to not have any remote, because we can\n        # still obtain the submodule from a future publication location, based on the\n        # parent\n        # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch)\n        # going git native instead\n        cmd = ['submodule', 'add', '--name', name]\n        if branch is not None:\n            cmd += ['-b', branch]\n        if url is None:\n            # repo must already exist locally\n            subm = GitRepo(op.join(self.path, path), create=False, init=False)\n            # check that it has a commit, and refuse\n            # to operate on it otherwise, or we would get a bastard\n            # submodule that cripples git operations\n            if not subm.get_hexsha():\n                raise InvalidGitRepositoryError(\n                    'cannot add subdataset {} with no commits'.format(subm))\n            # make an attempt to configure a submodule source URL based on the\n            # discovered remote configuration\n            remote, branch = subm.get_tracking_branch()\n            url = subm.get_remote_url(remote) if remote else None\n\n        if url is None:\n            # had no luck with a remote URL\n            if not isabs(path):\n                # need to recode into a relative path \"URL\" in POSIX\n                # style, even on windows\n                url = posixpath.join(curdir, posix_relpath(path))\n            else:\n                url = path\n        cmd += [url, Path(path).as_posix()]\n        self.call_git(cmd)\n        # record dataset ID if possible for comprehesive metadata on\n        # dataset components within the dataset itself\n        subm_id = GitRepo(op.join(self.path, path)).config.get(\n            'datalad.dataset.id', None)\n        if subm_id:\n            self.call_git(\n                ['config', '--file', '.gitmodules', '--replace-all',\n                 'submodule.{}.datalad-id'.format(name), subm_id])\n        # ensure supported setup\n        _fixup_submodule_dotgit_setup(self, path)\n        # TODO: return value\n\n    def deinit_submodule(self, path, **kwargs):\n        \"\"\"Deinit a submodule\n\n        Parameters\n        ----------\n        path: str\n            path to the submodule; relative to `self.path`\n        kwargs:\n            see `__init__`\n        \"\"\"\n\n        self._git_custom_command(path,\n                                 ['git', 'submodule', 'deinit'] +\n                                 to_options(**kwargs))\n        # TODO: return value\n\n    def update_submodule(self, path, mode='checkout', init=False):\n        \"\"\"Update a registered submodule.\n\n        This will make the submodule match what the superproject expects by\n        cloning missing submodules and updating the working tree of the\n        submodules. The \"updating\" can be done in several ways depending\n        on the value of submodule.<name>.update configuration variable, or\n        the `mode` argument.\n\n        Parameters\n        ----------\n        path : str\n          Identifies which submodule to operate on by it's repository-relative\n          path.\n        mode : {checkout, rebase, merge}\n          Update procedure to perform. 'checkout': the commit recorded in the\n          superproject will be checked out in the submodule on a detached HEAD;\n          'rebase': the current branch of the submodule will be rebased onto\n          the commit recorded in the superproject; 'merge': the commit recorded\n          in the superproject will be merged into the current branch in the\n          submodule.\n        init : bool\n          If True, initialize all submodules for which \"git submodule init\" has\n          not been called so far before updating.\n          Primarily provided for internal purposes and should not be used directly\n          since would result in not so annex-friendly .git symlinks/references\n          instead of full featured .git/ directories in the submodules\n        \"\"\"\n        if GitRepo.is_valid_repo(self.pathobj / path):\n            subrepo = GitRepo(self.pathobj / path, create=False)\n            subbranch = subrepo.get_active_branch() if subrepo else None\n            subbranch_hexsha = subrepo.get_hexsha(subbranch) if subrepo else None\n        else:\n            subrepo = None\n            subbranch = None\n            subbranch_hexsha = None\n\n        cmd = ['git', 'submodule', 'update', '--%s' % mode]\n        if init:\n            cmd.append('--init')\n            subgitpath = opj(self.path, path, '.git')\n            if not exists(subgitpath):\n                # TODO:  wouldn't with --init we get all those symlink'ed .git/?\n                # At least let's warn\n                lgr.warning(\n                    \"Do not use update_submodule with init=True to avoid git creating \"\n                    \"symlinked .git/ directories in submodules\"\n                )\n            #  yoh: I thought I saw one recently but thought it was some kind of\n            #  an artifact from running submodule update --init manually at\n            #  some point, but looking at this code now I worry that it was not\n        self._git_custom_command(path, cmd)\n\n        if not init:\n            return\n\n        # track branch originally cloned, only if we had a valid repo at the start\n        updated_subbranch = subrepo.get_active_branch() if subrepo else None\n        if subbranch and not updated_subbranch:\n            # got into 'detached' mode\n            # trace if current state is a predecessor of the branch_hexsha\n            lgr.debug(\n                \"Detected detached HEAD after updating submodule %s which was \"\n                \"in %s branch before\", self.path, subbranch)\n            detached_hexsha = subrepo.get_hexsha()\n            if subrepo.get_merge_base(\n                    [subbranch_hexsha, detached_hexsha]) == detached_hexsha:\n                # TODO: config option?\n                # in all likely event it is of the same branch since\n                # it is an ancestor -- so we could update that original branch\n                # to point to the state desired by the submodule, and update\n                # HEAD to point to that location\n                lgr.info(\n                    \"Submodule HEAD got detached. Resetting branch %s to point \"\n                    \"to %s. Original location was %s\",\n                    subbranch, detached_hexsha[:8], subbranch_hexsha[:8]\n                )\n                branch_ref = 'refs/heads/%s' % subbranch\n                subrepo.update_ref(branch_ref, detached_hexsha)\n                assert(subrepo.get_hexsha(subbranch) == detached_hexsha)\n                subrepo.update_ref('HEAD', branch_ref, symbolic=True)\n                assert(subrepo.get_active_branch() == subbranch)\n            else:\n                lgr.warning(\n                    \"%s has a detached HEAD since cloned branch %s has another common ancestor with %s\",\n                    subrepo.path, subbranch, detached_hexsha[:8]\n                )\n        # TODO: return value\n\n    def update_ref(self, ref, value, symbolic=False):\n        \"\"\"Update the object name stored in a ref \"safely\".\n\n        Just a shim for `git update-ref` call if not symbolic, and\n        `git symbolic-ref` if symbolic\n\n        Parameters\n        ----------\n        ref : str\n          Reference, such as `ref/heads/BRANCHNAME` or HEAD.\n        value : str\n          Value to update to, e.g. hexsha of a commit when updating for a\n          branch ref, or branch ref if updating HEAD\n        symbolic : None\n          To instruct if ref is symbolic, e.g. should be used in case of\n          ref=HEAD\n        \"\"\"\n        self._git_custom_command(\n            '',\n            ['git', 'symbolic-ref' if symbolic else 'update-ref', ref, value]\n        )\n\n    def tag(self, tag, message=None, commit=None, options=None):\n        \"\"\"Tag a commit\n\n        Parameters\n        ----------\n        tag : str\n          Custom tag label. Must be a valid tag name.\n        message : str, optional\n          If provided, adds ['-m', <message>] to the list of `git tag`\n          arguments.\n        commit : str, optional\n          If provided, will be appended as last argument to the `git tag` call,\n          and can be used to identify the commit that shall be tagged, if\n          not HEAD.\n        options : list, optional\n          Additional command options, inserted prior a potential `commit`\n          argument.\n        \"\"\"\n        # TODO: call in save.py complains about extensive logging. When does it\n        # happen in what way? Figure out, whether to just silence it or raise or\n        # whatever else.\n        args = ['tag']\n        if message:\n            args += ['-m', message]\n        if options is not None:\n            args.extend(options)\n        args.append(tag)\n        if commit:\n            args.append(commit)\n        self.call_git(args)\n\n    def get_tags(self, output=None):\n        \"\"\"Get list of tags\n\n        Parameters\n        ----------\n        output : str, optional\n          If given, limit the return value to a list of values matching that\n          particular key of the tag properties.\n\n        Returns\n        -------\n        list\n          Each item is a dictionary with information on a tag. At present\n          this includes 'hexsha', and 'name', where the latter is the string\n          label of the tag, and the former the hexsha of the object the tag\n          is attached to. The list is sorted by the creator date (committer\n          date for lightweight tags and tagger date for annotated tags), with\n          the most recent commit being the last element.\n        \"\"\"\n        tags = [\n            dict(\n                name=t['refname:strip=2'],\n                hexsha=t['object'] if t['object'] else t['objectname'],\n            )\n            for t in self.for_each_ref_(\n                fields=['refname:strip=2', 'objectname', 'object'],\n                pattern='refs/tags',\n                sort='creatordate')\n        ]\n        if output:\n            return [t[output] for t in tags]\n        else:\n            return tags\n\n    def describe(self, commitish=None, **kwargs):\n        \"\"\" Quick and dirty implementation to call git-describe\n\n        Parameters\n        ----------\n        kwargs:\n            transformed to cmdline options for git-describe;\n            see __init__ for description of the transformation\n        \"\"\"\n        # TODO: be more precise what failure to expect when and raise actual\n        # errors\n        cmd = ['git', 'describe'] + to_options(**kwargs)\n        if commitish is not None:\n            cmd.append(commitish)\n        try:\n            describe, outerr = self._git_custom_command(\n                [],\n                cmd,\n                expect_fail=True)\n            return describe.strip()\n        # TODO: WTF \"catch everything\"?\n        except:\n            return None\n\n    def get_tracking_branch(self, branch=None, remote_only=False):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        Parameters\n        ----------\n        branch: str\n            local branch to look up. If none is given, active branch is used.\n        remote_only : bool\n            Don't return a value if the upstream remote is set to \".\" (meaning\n            this repository).\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n        if branch is None:\n            branch = self.get_active_branch()\n            if branch is None:\n                return None, None\n\n        track_remote = self.config.get('branch.{0}.remote'.format(branch), None)\n        if remote_only and track_remote == \".\":\n            return None, None\n        track_branch = self.config.get('branch.{0}.merge'.format(branch), None)\n        return track_remote, track_branch\n\n    @property\n    def count_objects(self):\n        \"\"\"return dictionary with count, size(in KiB) information of git objects\n        \"\"\"\n\n        count_cmd = ['git', 'count-objects', '-v']\n        count_str, err = self._git_custom_command('', count_cmd)\n        count = {key: int(value)\n                 for key, value in [item.split(': ')\n                                    for item in count_str.split('\\n')\n                                    if len(item.split(': ')) == 2]}\n        return count\n\n    def get_git_attributes(self):\n        \"\"\"Query gitattributes which apply to top level directory\n\n        It is a thin compatibility/shortcut wrapper around more versatile\n        get_gitattributes which operates on a list of paths and returns\n        a dictionary per each path\n\n        Returns\n        -------\n        dict:\n          a dictionary with attribute name and value items relevant for the\n          top ('.') directory of the repository, and thus most likely the\n          default ones (if not overwritten with more rules) for all files within\n          repo.\n        \"\"\"\n        return self.get_gitattributes('.')['.']\n\n    def get_gitattributes(self, path, index_only=False):\n        \"\"\"Query gitattributes for one or more paths\n\n        Parameters\n        ----------\n        path: path or list\n          Path(s) to query. Paths may be relative or absolute.\n        index_only: bool\n          Flag whether to consider only gitattribute setting that are reflected\n          in the repository index, not just in the work tree content.\n\n        Returns\n        -------\n        dict:\n          Each key is a queried path (always relative to the repostiory root),\n          each value is a dictionary with attribute\n          name and value items. Attribute values are either True or False,\n          for set and unset attributes, or are the literal attribute value.\n        \"\"\"\n        path = ensure_list(path)\n        cmd = [\"git\", \"check-attr\", \"-z\", \"--all\"]\n        if index_only:\n            cmd.append('--cached')\n        stdout, stderr = self._git_custom_command(path, cmd)\n        # make sure we have one entry for each query path to\n        # simplify work with the result\n        attributes = {_normalize_path(self.path, p): {} for p in path}\n        attr = []\n        for item in stdout.split('\\0'):\n            attr.append(item)\n            if len(attr) < 3:\n                continue\n            # we have a full record\n            p, name, value = attr\n            attrs = attributes[p]\n            attrs[name] = \\\n                True if value == 'set' else False if value == 'unset' else value\n            # done, reset item\n            attr = []\n        return attributes\n\n    def set_gitattributes(self, attrs, attrfile='.gitattributes', mode='a'):\n        \"\"\"Set gitattributes\n\n        By default appends additional lines to `attrfile`. Note, that later\n        lines in `attrfile` overrule earlier ones, which may or may not be\n        what you want. Set `mode` to 'w' to replace the entire file by\n        what you provided in `attrs`.\n\n        Parameters\n        ----------\n        attrs : list\n          Each item is a 2-tuple, where the first element is a path pattern,\n          and the second element is a dictionary with attribute key/value\n          pairs. The attribute dictionary must use the same semantics as those\n          returned by `get_gitattributes()`. Path patterns can use absolute paths,\n          in which case they will be normalized relative to the directory\n          that contains the target .gitattributes file (see `attrfile`).\n        attrfile: path\n          Path relative to the repository root of the .gitattributes file the\n          attributes shall be set in.\n        mode: str\n          'a' to append .gitattributes, 'w' to replace it\n        \"\"\"\n\n        git_attributes_file = op.join(self.path, attrfile)\n        attrdir = op.dirname(git_attributes_file)\n        if not op.exists(attrdir):\n            os.makedirs(attrdir)\n        with open(git_attributes_file, mode) as f:\n            for pattern, attr in sorted(attrs, key=lambda x: x[0]):\n                # normalize the pattern relative to the target .gitattributes file\n                npath = _normalize_path(\n                    op.join(self.path, op.dirname(attrfile)), pattern)\n                # paths in gitattributes always have to be POSIX\n                npath = Path(npath).as_posix()\n                attrline = u''\n                if npath.count(' '):\n                    # quote patterns with spaces\n                    attrline += u'\"{}\"'.format(npath.replace('\"', '\\\\\"'))\n                else:\n                    attrline += npath\n                for a in sorted(attr):\n                    val = attr[a]\n                    if val is True:\n                        attrline += ' {}'.format(a)\n                    elif val is False:\n                        attrline += ' -{}'.format(a)\n                    else:\n                        attrline += ' {}={}'.format(a, val)\n                f.write('\\n{}'.format(attrline))\n\n    def get_content_info(self, paths=None, ref=None, untracked='all',\n                         eval_file_type=True):\n        \"\"\"Get identifier and type information from repository content.\n\n        This is simplified front-end for `git ls-files/tree`.\n\n        Both commands differ in their behavior when queried about subdataset\n        paths. ls-files will not report anything, ls-tree will report on the\n        subdataset record. This function uniformly follows the behavior of\n        ls-tree (report on the respective subdataset mount).\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath)\n          Specific paths, relative to the resolved repository root, to query\n          info for. Paths must be normed to match the reporting done by Git,\n          i.e. no parent dir components (ala \"some/../this\").\n          If none are given, info is reported for all content.\n        ref : gitref or None\n          If given, content information is retrieved for this Git reference\n          (via ls-tree), otherwise content information is produced for the\n          present work tree (via ls-files). With a given reference, the\n          reported content properties also contain a 'bytesize' record,\n          stating the size of a file in bytes.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when no `ref` was given:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_file_type : bool\n          If True, inspect file type of untracked files, and report annex\n          symlink pointers as type 'file'. This convenience comes with a\n          cost; disable to get faster performance if this information\n          is not needed.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n\n            Note that the reported type will not always match the type of\n            content committed to Git, rather it will reflect the nature\n            of the content minus platform/mode-specifics. For example,\n            a symlink to a locked annexed file on Unix will have a type\n            'file', reported, while a symlink to a file in Git or directory\n            will be of type 'symlink'.\n\n          `gitshasum`\n            SHASUM of the item as tracked by Git, or None, if not\n            tracked. This could be different from the SHASUM of the file\n            in the worktree, if it was modified.\n\n        Raises\n        ------\n        ValueError\n          In case of an invalid Git reference (e.g. 'HEAD' in an empty\n          repository)\n        \"\"\"\n        lgr.debug('%s.get_content_info(...)', self)\n        # TODO limit by file type to replace code in subdatasets command\n        info = OrderedDict()\n\n        if paths:\n            # path matching will happen against what Git reports\n            # and Git always reports POSIX paths\n            # any incoming path has to be relative already, so we can simply\n            # convert unconditionally\n            paths = [ut.PurePosixPath(p) for p in paths]\n\n        path_strs = list(map(str, paths)) if paths else None\n\n        # this will not work in direct mode, but everything else should be\n        # just fine\n        if not ref:\n            # make sure no operations are pending before we figure things\n            # out in the worktree\n            self.precommit()\n\n            # --exclude-standard will make sure to honor and standard way\n            # git can be instructed to ignore content, and will prevent\n            # crap from contaminating untracked file reports\n            cmd = ['git', 'ls-files',\n                   '--stage', '-z', '-d', '-m', '--exclude-standard']\n            # untracked report mode, using labels from `git diff` option style\n            if untracked == 'all':\n                cmd.append('-o')\n            elif untracked == 'normal':\n                cmd += ['-o', '--directory', '--no-empty-directory']\n            elif untracked == 'no':\n                pass\n            else:\n                raise ValueError(\n                    'unknown value for `untracked`: {}'.format(untracked))\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) (?P<sha>.*) (.*)\\t(?P<fname>.*)$')\n\n            if path_strs:\n                # we need to get their within repo elements since ls-tree\n                # for paths within submodules returns nothing!\n                # see https://public-inbox.org/git/20190703193305.GF21553@hopa.kiewit.dartmouth.edu/T/#u\n                submodules = [s.path for s in self.get_submodules()]\n                path_strs = get_parent_paths(path_strs, submodules)\n        else:\n            cmd = ['git', 'ls-tree', ref, '-z', '-r', '--full-tree', '-l']\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) ([a-z]*) (?P<sha>[^ ]*) [\\s]*(?P<size>[0-9-]+)\\t(?P<fname>.*)$')\n\n        lgr.debug('Query repo: %s', cmd)\n        try:\n            stdout, stderr = self._git_custom_command(\n                path_strs,\n                cmd,\n                log_stderr=True,\n                log_stdout=True,\n                # not sure why exactly, but log_online has to be false!\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            if \"fatal: Not a valid object name\" in str(exc):\n                raise InvalidGitReferenceError(ref)\n            raise\n        lgr.debug('Done query repo: %s', cmd)\n\n        if not eval_file_type:\n            _get_link_target = None\n        elif ref:\n            def _read_symlink_target_from_catfile(lines):\n                # it is always the second line, all checks done upfront\n                header = lines.readline()\n                if header.rstrip().endswith('missing'):\n                    # something we do not know about, should not happen\n                    # in real use, but guard against to avoid stalling\n                    return ''\n                return lines.readline().rstrip()\n\n            _get_link_target = BatchedCommand(\n                ['git', 'cat-file', '--batch'],\n                path=self.path,\n                output_proc=_read_symlink_target_from_catfile,\n            )\n        else:\n            def try_readlink(path):\n                try:\n                    return os.readlink(path)\n                except OSError:\n                    # readlink will fail if the symlink reported by ls-files is\n                    # not in the working tree (it could be removed or\n                    # unlocked). Fall back to a slower method.\n                    return op.realpath(path)\n\n            _get_link_target = try_readlink\n\n        try:\n            self._get_content_info_line_helper(\n                ref,\n                info,\n                stdout.split('\\0'),\n                props_re,\n                _get_link_target)\n        finally:\n            if ref and _get_link_target:\n                # cancel batch process\n                _get_link_target.close()\n\n        lgr.debug('Done %s.get_content_info(...)', self)\n        return info\n\n    def _get_content_info_line_helper(self, ref, info, lines,\n                                      props_re, get_link_target):\n        \"\"\"Internal helper of get_content_info() to parse Git output\"\"\"\n        mode_type_map = {\n            '100644': 'file',\n            '100755': 'file',\n            '120000': 'symlink',\n            '160000': 'dataset',\n        }\n        for line in lines:\n            if not line:\n                continue\n            inf = {}\n            props = props_re.match(line)\n            if not props:\n                # not known to Git, but Git always reports POSIX\n                path = ut.PurePosixPath(line)\n                inf['gitshasum'] = None\n            else:\n                # again Git reports always in POSIX\n                path = ut.PurePosixPath(props.group('fname'))\n\n            # revisit the file props after this path has not been rejected\n            if props:\n                inf['gitshasum'] = props.group('sha')\n                inf['type'] = mode_type_map.get(\n                    props.group('type'), props.group('type'))\n                if get_link_target and inf['type'] == 'symlink' and \\\n                        ((ref is None and '.git/annex/objects' in \\\n                          ut.Path(\n                            get_link_target(str(self.pathobj / path))\n                          ).as_posix()) or \\\n                         (ref and \\\n                          '.git/annex/objects' in get_link_target(\n                              u'{}:{}'.format(\n                                  ref, str(path))))\n                        ):\n                    # report annex symlink pointers as file, their\n                    # symlink-nature is a technicality that is dependent\n                    # on the particular mode annex is in\n                    inf['type'] = 'file'\n\n                if ref and inf['type'] == 'file':\n                    inf['bytesize'] = int(props.group('size'))\n\n            # join item path with repo path to get a universally useful\n            # path representation with auto-conversion and tons of other\n            # stuff\n            path = self.pathobj.joinpath(path)\n            if 'type' not in inf:\n                # be nice and assign types for untracked content\n                inf['type'] = 'symlink' if path.is_symlink() \\\n                    else 'directory' if path.is_dir() else 'file'\n            info[path] = inf\n\n    def status(self, paths=None, untracked='all', eval_submodule_state='full'):\n        \"\"\"Simplified `git status` equivalent.\n\n        Parameters\n        ----------\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list. If a query path points\n          into a subdataset, a report is made on the subdataset record\n          within the queried dataset only (no recursion).\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        lgr.debug('Query status of %r for %s paths',\n                  self, len(paths) if paths else 'all')\n        return self.diffstatus(\n            fr='HEAD' if self.get_hexsha() else None,\n            to=None,\n            paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state)\n\n    def diff(self, fr, to, paths=None, untracked='all',\n             eval_submodule_state='full'):\n        \"\"\"Like status(), but reports changes between to arbitrary revisions\n\n        Parameters\n        ----------\n        fr : str or None\n          Revision specification (anything that Git understands). Passing\n          `None` considers anything in the target state as new.\n        to : str or None\n          Revision specification (anything that Git understands), or None\n          to compare to the state of the work tree.\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when `to` is None:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        return {k: v for k, v in self.diffstatus(\n            fr=fr, to=to, paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state).items()\n            if v.get('state', None) != 'clean'}\n\n    def diffstatus(self, fr, to, paths=None, untracked='all',\n                   eval_submodule_state='full', eval_file_type=True,\n                   _cache=None):\n        \"\"\"Like diff(), but reports the status of 'clean' content too\"\"\"\n        return self._diffstatus(\n            fr, to, paths, untracked, eval_submodule_state, eval_file_type,\n            _cache)\n\n    def _diffstatus(self, fr, to, paths, untracked, eval_state,\n                    eval_file_type, _cache):\n        \"\"\"Just like diffstatus(), but supports an additional evaluation\n        state 'global'. If given, it will return a single 'modified'\n        (vs. 'clean') state label for the entire repository, as soon as\n        it can.\"\"\"\n        def _get_cache_key(label, paths, ref, untracked=None):\n            return self.path, label, tuple(paths) if paths else None, \\\n                ref, untracked\n\n        if _cache is None:\n            _cache = {}\n\n        if paths:\n            # at this point we must normalize paths to the form that\n            # Git would report them, to easy matching later on\n            paths = [ut.Path(p) for p in paths]\n            paths = [\n                p.relative_to(self.pathobj) if p.is_absolute() else p\n                for p in paths\n            ]\n\n        # TODO report more info from get_content_info() calls in return\n        # value, those are cheap and possibly useful to a consumer\n        # we need (at most) three calls to git\n        if to is None:\n            # everything we know about the worktree, including os.stat\n            # for each file\n            key = _get_cache_key('ci', paths, None, untracked)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=None, untracked=untracked,\n                    eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we want Git to tell us what it considers modified and avoid\n            # reimplementing logic ourselves\n            key = _get_cache_key('mod', paths, None)\n            if key in _cache:\n                modified = _cache[key]\n            else:\n                modified = set(\n                    self.pathobj.joinpath(ut.PurePosixPath(p))\n                    for p in self._git_custom_command(\n                        # low-level code cannot handle pathobjs\n                        [str(p) for p in paths] if paths else None,\n                        ['git', 'ls-files', '-z', '-m'])[0].split('\\0')\n                    if p)\n                _cache[key] = modified\n        else:\n            key = _get_cache_key('ci', paths, to)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=to, eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we do not need worktree modification detection in this case\n            modified = None\n        # origin state\n        key = _get_cache_key('ci', paths, fr)\n        if key in _cache:\n            from_state = _cache[key]\n        else:\n            if fr:\n                from_state = self.get_content_info(\n                    paths=paths, ref=fr, eval_file_type=eval_file_type)\n            else:\n                # no ref means from nothing\n                from_state = {}\n            _cache[key] = from_state\n\n        status = OrderedDict()\n        for f, to_state_r in to_state.items():\n            props = None\n            if f not in from_state:\n                # this is new, or rather not known to the previous state\n                props = dict(\n                    state='added' if to_state_r['gitshasum'] else 'untracked',\n                )\n                if 'type' in to_state_r:\n                    props['type'] = to_state_r['type']\n            elif to_state_r['gitshasum'] == from_state[f]['gitshasum'] and \\\n                    (modified is None or f not in modified):\n                if to_state_r['type'] != 'dataset':\n                    # no change in git record, and no change on disk\n                    props = dict(\n                        # at this point we know that the reported object ids\n                        # for this file are identical in the to and from\n                        # records.  If to is None, we're comparing to the\n                        # working tree and a deleted file will still have an\n                        # identical id, so we need to check whether the file is\n                        # gone before declaring it clean. This working tree\n                        # check is irrelevant and wrong if to is a ref.\n                        state='clean' if to is not None or (f.exists() or \\\n                              f.is_symlink()) else 'deleted',\n                        type=to_state_r['type'],\n                    )\n                else:\n                    # a dataset\n                    props = dict(type=to_state_r['type'])\n                    if to is not None:\n                        # we can only be confident without looking\n                        # at the worktree, if we compare to a recorded\n                        # state\n                        props['state'] = 'clean'\n                    else:\n                        # report the shasum that we know, for further\n                        # wrangling of subdatasets below\n                        props['gitshasum'] = to_state_r['gitshasum']\n                        props['prev_gitshasum'] = from_state[f]['gitshasum']\n            else:\n                # change in git record, or on disk\n                props = dict(\n                    # TODO we could have a new file that is already staged\n                    # but had subsequent modifications done to it that are\n                    # unstaged. Such file would presently show up as 'added'\n                    # ATM I think this is OK, but worth stating...\n                    state='modified' if f.exists() or \\\n                    f.is_symlink() else 'deleted',\n                    # TODO record before and after state for diff-like use\n                    # cases\n                    type=to_state_r['type'],\n                )\n            state = props.get('state', None)\n            if eval_state == 'global' and \\\n                    state not in ('clean', None):\n                # any modification means globally 'modified'\n                return 'modified'\n            if state in ('clean', 'added', 'modified'):\n                props['gitshasum'] = to_state_r['gitshasum']\n                if 'bytesize' in to_state_r:\n                    # if we got this cheap, report it\n                    props['bytesize'] = to_state_r['bytesize']\n                elif props['state'] == 'clean' and 'bytesize' in from_state[f]:\n                    # no change, we can take this old size info\n                    props['bytesize'] = from_state[f]['bytesize']\n            if state in ('clean', 'modified', 'deleted'):\n                props['prev_gitshasum'] = from_state[f]['gitshasum']\n            status[f] = props\n\n        for f, from_state_r in from_state.items():\n            if f not in to_state:\n                # we new this, but now it is gone and Git is not complaining\n                # about it being missing -> properly deleted and deletion\n                # stages\n                status[f] = dict(\n                    state='deleted',\n                    type=from_state_r['type'],\n                    # report the shasum to distinguish from a plainly vanished\n                    # file\n                    gitshasum=from_state_r['gitshasum'],\n                )\n                if eval_state == 'global':\n                    return 'modified'\n\n        if to is not None or eval_state == 'no':\n            # if we have `to` we are specifically comparing against\n            # a recorded state, and this function only attempts\n            # to label the state of a subdataset, not investigate\n            # specifically what the changes in subdatasets are\n            # this is done by a high-level command like rev-diff\n            # so the comparison within this repo and the present\n            # `state` label are all we need, and they are done already\n            if eval_state == 'global':\n                return 'clean'\n            else:\n                return status\n\n        # loop over all subdatasets and look for additional modifications\n        for f, st in status.items():\n            f = str(f)\n            if 'state' in st or not st['type'] == 'dataset':\n                # no business here\n                continue\n            if not GitRepo.is_valid_repo(f):\n                # submodule is not present, no chance for a conflict\n                st['state'] = 'clean'\n                continue\n            # we have to recurse into the dataset and get its status\n            subrepo = GitRepo(f)\n            subrepo_commit = subrepo.get_hexsha()\n            st['gitshasum'] = subrepo_commit\n            # subdataset records must be labeled clean up to this point\n            # test if current commit in subdataset deviates from what is\n            # recorded in the dataset\n            st['state'] = 'modified' \\\n                if st['prev_gitshasum'] != subrepo_commit \\\n                else 'clean'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n            if eval_state == 'commit':\n                continue\n            # the recorded commit did not change, so we need to make\n            # a more expensive traversal\n            st['state'] = subrepo._diffstatus(\n                # we can use 'HEAD' because we know that the commit\n                # did not change. using 'HEAD' will facilitate\n                # caching the result\n                fr='HEAD',\n                to=None,\n                paths=None,\n                untracked=untracked,\n                eval_state='global',\n                eval_file_type=False,\n                _cache=_cache) if st['state'] == 'clean' else 'modified'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n\n        if eval_state == 'global':\n            return 'clean'\n        else:\n            return status\n\n    def _save_pre(self, paths, _status, **kwargs):\n        # helper to get an actionable status report\n        if paths is not None and not paths and not _status:\n            return\n        if _status is None:\n            if 'untracked' not in kwargs:\n                kwargs['untracked'] = 'normal'\n            status = self.status(\n                paths=paths,\n                **{k: kwargs[k] for k in kwargs\n                   if k in ('untracked', 'eval_submodule_state')})\n        else:\n            # we want to be able to add items down the line\n            # make sure to detach from prev. owner\n            status = _status.copy()\n        status = OrderedDict(\n            (k, v) for k, v in status.items()\n            if v.get('state', None) != 'clean'\n        )\n        return status\n\n    def get_staged_paths(self):\n        \"\"\"Returns a list of any stage repository path(s)\n\n        This is a rather fast call, as it will not depend on what is going on\n        in the worktree.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                ['git', 'diff', '--name-only', '--staged'],\n                cwd=self.path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                expect_fail=True)\n        except CommandError as e:\n            lgr.debug(exc_str(e))\n            stdout = ''\n        return [f for f in stdout.split('\\n') if f]\n\n    def _save_post(self, message, status, partial_commit):\n        # helper to commit changes reported in status\n\n        # TODO remove pathobj stringification when commit() can\n        # handle it\n        to_commit = [str(f.relative_to(self.pathobj))\n                     for f, props in status.items()] \\\n                    if partial_commit else None\n        if not partial_commit or to_commit:\n            # we directly call GitRepo.commit() to avoid a whole slew\n            # if direct-mode safeguards and workarounds in the AnnexRepo\n            # implementation (which also run an additional dry-run commit\n            GitRepo.commit(\n                self,\n                files=to_commit,\n                msg=message,\n                options=None,\n                # do not raise on empty commit\n                # it could be that the `add` in this save-cycle has already\n                # brought back a 'modified' file into a clean state\n                careless=True,\n            )\n\n    def save(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Save dataset content.\n\n        Parameters\n        ----------\n        message : str or None\n          A message to accompany the changeset in the log. If None,\n          a default message is used.\n        paths : list or None\n          Any content with path matching any of the paths given in this\n          list will be saved. Matching will be performed against the\n          dataset status (GitRepo.status()), or a custom status provided\n          via `_status`. If no paths are provided, ALL non-clean paths\n          present in the repo status or `_status` will be saved.\n        _status : dict or None\n          If None, Repo.status() will be queried for the given `ds`. If\n          a dict is given, its content will be used as a constraint.\n          For example, to save only modified content, but no untracked\n          content, set `paths` to None and provide a `_status` that has\n          no entries for untracked content.\n        **kwargs :\n          Additional arguments that are passed to underlying Repo methods.\n          Supported:\n\n          - git : bool (passed to Repo.add()\n          - eval_submodule_state : {'full', 'commit', 'no'}\n            passed to Repo.status()\n          - untracked : {'no', 'normal', 'all'} - passed to Repo.satus()\n        \"\"\"\n        return list(\n            self.save_(\n                message=message,\n                paths=paths,\n                _status=_status,\n                **kwargs\n            )\n        )\n\n    def save_(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Like `save()` but working as a generator.\"\"\"\n        from datalad.interface.results import get_status_dict\n\n        status = self._save_pre(paths, _status, **kwargs)\n        if not status:\n            # all clean, nothing todo\n            lgr.debug('Nothing to save in %r, exiting early', self)\n            return\n\n        # three things are to be done:\n        # - remove (deleted if not already staged)\n        # - add (modified/untracked)\n        # - commit (with all paths that have been touched, to bypass\n        #   potential pre-staged bits)\n\n        need_partial_commit = True if self.get_staged_paths() else False\n\n        # remove first, because removal of a subds would cause a\n        # modification of .gitmodules to be added to the todo list\n        to_remove = [\n            # TODO remove pathobj stringification when delete() can\n            # handle it\n            str(f.relative_to(self.pathobj))\n            for f, props in status.items()\n            if props.get('state', None) == 'deleted' and\n            # staged deletions have a gitshasum reported for them\n            # those should not be processed as git rm will error\n            # due to them being properly gone already\n            not props.get('gitshasum', None)]\n        vanished_subds = any(\n            props.get('type', None) == 'dataset' and\n            props.get('state', None) == 'deleted'\n            for f, props in status.items())\n        if to_remove:\n            for r in self.remove(\n                    to_remove,\n                    # we would always see individual files\n                    recursive=False):\n                # TODO normalize result\n                yield get_status_dict(\n                    action='delete',\n                    refds=self.pathobj,\n                    # TODO make remove() report the type\n                    # for now it claims to report on files only\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r)),\n                    # make remove() report on failures too\n                    status='ok',\n                    logger=lgr)\n\n        # TODO this additonal query should not be, base on status as given\n        # if anyhow possible, however, when paths are given, status may\n        # not contain all required information. In case of path=None AND\n        # _status=None, we should be able to avoid this, because\n        # status should have the full info already\n        # looks for contained repositories\n        added_submodule = False\n        untracked_dirs = [f.relative_to(self.pathobj)\n                          for f, props in status.items()\n                          if props.get('state', None) == 'untracked' and\n                          props.get('type', None) == 'directory']\n        to_add_submodules = []\n        if untracked_dirs:\n            to_add_submodules = [sm for sm, sm_props in\n                self.get_content_info(\n                    untracked_dirs,\n                    ref=None,\n                    # request exhaustive list, so that everything that is\n                    # still reported as a directory must be its own repository\n                    untracked='all').items()\n                if sm_props.get('type', None) == 'directory']\n            to_add_submodules = _prune_deeper_repos(to_add_submodules)\n            for cand_sm in to_add_submodules:\n                branch = self.get_active_branch()\n                adjusted_match = ADJUSTED_BRANCH_EXPR.match(\n                    branch if branch else '')\n                try:\n                    self.add_submodule(\n                        str(cand_sm.relative_to(self.pathobj)),\n                        url=None,\n                        name=None,\n                        branch=adjusted_match.group('name') if adjusted_match\n                        else branch\n                    )\n                except (CommandError, InvalidGitRepositoryError) as e:\n                    yield get_status_dict(\n                        action='add_submodule',\n                        ds=self,\n                        path=self.pathobj / ut.PurePosixPath(cand_sm),\n                        status='error',\n                        message=e.stderr if hasattr(e, 'stderr')\n                        else ('not a Git repository: %s', exc_str(e)),\n                        logger=lgr)\n                    continue\n                # This mirrors the result structure yielded for\n                # to_stage_submodules below.\n                yield get_status_dict(\n                    action='add',\n                    refds=self.pathobj,\n                    type='file',\n                    key=None,\n                    path=self.pathobj / ut.PurePosixPath(cand_sm),\n                    status='ok',\n                    logger=lgr)\n                added_submodule = True\n        if not need_partial_commit:\n            # without a partial commit an AnnexRepo would ignore any submodule\n            # path in its add helper, hence `git add` them explicitly\n            to_stage_submodules = {\n                str(f.relative_to(self.pathobj)): props\n                for f, props in status.items()\n                if props.get('state', None) in ('modified', 'untracked')\n                and props.get('type', None) == 'dataset'}\n            if to_stage_submodules:\n                lgr.debug(\n                    '%i submodule path(s) to stage in %r %s',\n                    len(to_stage_submodules), self,\n                    to_stage_submodules\n                    if len(to_stage_submodules) < 10 else '')\n                for r in GitRepo._save_add(\n                        self,\n                        to_stage_submodules,\n                        git_opts=None):\n                    yield r\n\n        if added_submodule or vanished_subds:\n            # need to include .gitmodules in what needs saving\n            status[self.pathobj.joinpath('.gitmodules')] = dict(\n                type='file', state='modified')\n            if hasattr(self, 'annexstatus') and not kwargs.get('git', False):\n                # we cannot simply hook into the coming add-call\n                # as this would go to annex, so make a dedicted git-add\n                # call to ensure .gitmodules is not annexed\n                # in any normal DataLad dataset .gitattributes will\n                # prevent this, but in a plain repo it won't\n                # https://github.com/datalad/datalad/issues/3306\n                for r in GitRepo._save_add(\n                        self,\n                        {op.join(self.path, '.gitmodules'): None}):\n                    yield r\n        to_add = {\n            # TODO remove pathobj stringification when add() can\n            # handle it\n            str(f.relative_to(self.pathobj)): props\n            for f, props in status.items()\n            if (props.get('state', None) in ('modified', 'untracked') and\n                f not in to_add_submodules)}\n        if to_add:\n            lgr.debug(\n                '%i path(s) to add to %s %s',\n                len(to_add), self, to_add if len(to_add) < 10 else '')\n            for r in self._save_add(\n                    to_add,\n                    git_opts=None,\n                    **{k: kwargs[k] for k in kwargs\n                       if k in (('git',) if hasattr(self, 'annexstatus')\n                                else tuple())}):\n                yield r\n\n        self._save_post(message, status, need_partial_commit)\n        # TODO yield result for commit, prev helper checked hexsha pre\n        # and post...\n\n    def _save_add(self, files, git_opts=None):\n        \"\"\"Simple helper to add files in save()\"\"\"\n        from datalad.interface.results import get_status_dict\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                list(files.keys()),\n                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']\n            )\n            # get all the entries\n            for r in self._process_git_get_output(*add_out):\n                yield get_status_dict(\n                    action=r.get('command', 'add'),\n                    refds=self.pathobj,\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r['file']))\n                    if 'file' in r else None,\n                    status='ok' if r.get('success', None) else 'error',\n                    key=r.get('key', None),\n                    # while there is no git-annex underneath here, we\n                    # tend to fake its behavior, so we can also support\n                    # this type of messaging\n                    message='\\n'.join(r['error-messages'])\n                    if 'error-messages' in r else None,\n                    logger=lgr)\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n\n# TODO\n# remove submodule: nope, this is just deinit_submodule + remove\n# status?\n\n\ndef _fixup_submodule_dotgit_setup(ds, relativepath):\n    \"\"\"Implementation of our current of .git in a subdataset\n\n    Each subdataset/module has its own .git directory where a standalone\n    repository would have it. No gitdir files, no symlinks.\n    \"\"\"\n    # move .git to superrepo's .git/modules, remove .git, create\n    # .git-file\n    path = opj(ds.path, relativepath)\n    subds_dotgit = opj(path, \".git\")\n\n    repo = GitRepo(path, create=False)\n    if repo.dot_git.parent == repo.pathobj:\n        # this is what we want\n        return\n\n    # first we want to remove any conflicting worktree setup\n    # done by git to find the checkout at the mountpoint of the\n    # submodule, if we keep that, any git command will fail\n    # after we move .git\n    # Ben: Shouldn't we re-setup a possible worktree afterwards?\n    repo.config.unset('core.worktree', where='local')\n    # what we have here is some kind of reference, remove and\n    # replace by the target\n    os.remove(subds_dotgit)\n    # make absolute\n    src_dotgit = str(repo.dot_git)\n    # move .git\n    from os import rename, listdir, rmdir\n    ensure_dir(subds_dotgit)\n    for dot_git_entry in listdir(src_dotgit):\n        rename(opj(src_dotgit, dot_git_entry),\n               opj(subds_dotgit, dot_git_entry))\n    assert not listdir(src_dotgit)\n    rmdir(src_dotgit)\n",
            "header": {
                "file": {
                    "path": "datalad/support/gitrepo.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "0764e6c3f",
                "new_commit_hash": "098260c77",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/gitrepo.py"
            },
            "plus_file": {
                "path": "b/datalad/support/gitrepo.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 137,
                            "length": 9
                        },
                        "new_range": {
                            "start": 137,
                            "length": 16
                        },
                        "section": "default_git_odbt = gitpy.GitCmdObjectDB"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "# log Exceptions from git commands.",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def to_options(**kwargs):",
                                "type": "deleted"
                            },
                            {
                                "content": "def to_options(split_single_char_options=True, **kwargs):",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Transform keyword arguments into a list of cmdline options",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    Imported from GitPython.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Original copyright:",
                                "type": "added"
                            },
                            {
                                "content": "        Copyright (C) 2008, 2009 Michael Trier and contributors",
                                "type": "added"
                            },
                            {
                                "content": "    Original license:",
                                "type": "added"
                            },
                            {
                                "content": "        BSD 3-Clause \"New\" or \"Revised\" License",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Parameters",
                                "type": "context"
                            },
                            {
                                "content": "    ----------",
                                "type": "context"
                            },
                            {
                                "content": "    split_single_char_options: bool",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/gitrepo.py",
                            "type": "function",
                            "name": "to_options",
                            "content": "def to_options(**kwargs):\n    \"\"\"Transform keyword arguments into a list of cmdline options\n\n    Parameters\n    ----------\n    split_single_char_options: bool\n\n    kwargs:\n\n    Returns\n    -------\n    list\n    \"\"\"\n    # TODO: borrow_docs!\n\n    return gitpy.Git().transform_kwargs(**kwargs)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 140,
                            "end_lineno": 155,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 150,
                            "length": 9
                        },
                        "new_range": {
                            "start": 157,
                            "length": 34
                        },
                        "section": "def to_options(**kwargs):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    -------",
                                "type": "context"
                            },
                            {
                                "content": "    list",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    # TODO: borrow_docs!",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "deleted"
                            },
                            {
                                "content": "    return gitpy.Git().transform_kwargs(**kwargs)",
                                "type": "deleted"
                            },
                            {
                                "content": "    def dashify(string):",
                                "type": "added"
                            },
                            {
                                "content": "        return string.replace('_', '-')",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    def transform_kwarg(name, value, split_single_char_options):",
                                "type": "added"
                            },
                            {
                                "content": "        if len(name) == 1:",
                                "type": "added"
                            },
                            {
                                "content": "            if value is True:",
                                "type": "added"
                            },
                            {
                                "content": "                return [\"-%s\" % name]",
                                "type": "added"
                            },
                            {
                                "content": "            elif value not in (False, None):",
                                "type": "added"
                            },
                            {
                                "content": "                if split_single_char_options:",
                                "type": "added"
                            },
                            {
                                "content": "                    return [\"-%s\" % name, \"%s\" % value]",
                                "type": "added"
                            },
                            {
                                "content": "                else:",
                                "type": "added"
                            },
                            {
                                "content": "                    return [\"-%s%s\" % (name, value)]",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            if value is True:",
                                "type": "added"
                            },
                            {
                                "content": "                return [\"--%s\" % dashify(name)]",
                                "type": "added"
                            },
                            {
                                "content": "            elif value is not False and value is not None:",
                                "type": "added"
                            },
                            {
                                "content": "                return [\"--%s=%s\" % (dashify(name), value)]",
                                "type": "added"
                            },
                            {
                                "content": "        return []",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    args = []",
                                "type": "added"
                            },
                            {
                                "content": "    kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0]))",
                                "type": "added"
                            },
                            {
                                "content": "    for k, v in kwargs.items():",
                                "type": "added"
                            },
                            {
                                "content": "        if isinstance(v, (list, tuple)):",
                                "type": "added"
                            },
                            {
                                "content": "            for value in v:",
                                "type": "added"
                            },
                            {
                                "content": "                args += transform_kwarg(k, value, split_single_char_options)",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "added"
                            },
                            {
                                "content": "            args += transform_kwarg(k, v, split_single_char_options)",
                                "type": "added"
                            },
                            {
                                "content": "    return args",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def _normalize_path(base_dir, path):",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/gitrepo.py",
                            "type": "function",
                            "name": "to_options",
                            "content": "def to_options(**kwargs):\n    \"\"\"Transform keyword arguments into a list of cmdline options\n\n    Parameters\n    ----------\n    split_single_char_options: bool\n\n    kwargs:\n\n    Returns\n    -------\n    list\n    \"\"\"\n    # TODO: borrow_docs!\n\n    return gitpy.Git().transform_kwargs(**kwargs)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 140,
                            "end_lineno": 155,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class GitRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import assert_is_instance\n\nimport logging\n\nimport os\nimport os.path as op\n\nimport sys\n\n\nfrom datalad import get_encoding_info\nfrom datalad.cmd import Runner\n\nfrom datalad.utils import (\n    chpwd,\n    getpwd,\n    on_windows,\n    rmtree,\n    unlink,\n)\nfrom datalad.tests.utils import (\n    assert_cwd_unchanged,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    create_tree,\n    eq_,\n    get_most_obscure_supported_name,\n    integration,\n    known_failure_windows,\n    local_testrepo_flavors,\n    neq_,\n    ok_,\n    ok_clean_git,\n    skip_if,\n    skip_if_no_network,\n    skip_if_on_windows,\n    skip_ssh,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_testrepos,\n    with_tree,\n)\nfrom datalad.tests.utils_testrepos import BasicAnnexTestRepo\n\nfrom datalad.dochelpers import exc_str\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.support.gitrepo import (\n    _normalize_path,\n    GitCommandError,\n    gitpy,\n    GitRepo,\n    guard_BadName,\n    InvalidGitRepositoryError,\n    normalize_paths,\n    NoSuchPathError,\n    to_options,\n)\nfrom datalad.support.exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    PathKnownToRepositoryError,\n)\nfrom datalad.support.external_versions import external_versions\nfrom datalad.support.protocol import ExecutionTimeProtocol\nfrom .utils import check_repo_deals_with_inode_change\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_invalid_path(path):\n    with chpwd(path):\n        assert_raises(ValueError, GitRepo, path=\"git://some/url\", create=True)\n        ok_(not op.exists(op.join(path, \"git:\")))\n        assert_raises(ValueError, GitRepo, path=\"file://some/relative/path\", create=True)\n        ok_(not op.exists(op.join(path, \"file:\")))\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_instance_from_clone(src, dst):\n\n    gr = GitRepo.clone(src, dst)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    assert_is_instance(gr.repo, gitpy.Repo,\n                       \"Failed to instantiate GitPython Repo object.\")\n    ok_(op.exists(op.join(dst, '.git')))\n\n    # do it again should raise GitCommandError since git will notice there's\n    # already a git-repo at that path and therefore can't clone to `dst`\n    # Note: Since GitRepo is now a WeakSingletonRepo, this is prevented from\n    # happening atm. Disabling for now:\n#    raise SkipTest(\"Disabled for RF: WeakSingletonRepo\")\n    with swallow_logs() as logs:\n        assert_raises(GitCommandError, GitRepo.clone, src, dst)\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\ndef test_GitRepo_instance_from_existing(path):\n\n    gr = GitRepo(path)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_instance_from_not_existing(path, path2):\n    # 1. create=False and path doesn't exist:\n    assert_raises(NoSuchPathError, GitRepo, path, create=False)\n    assert_false(op.exists(path))\n\n    # 2. create=False, path exists, but no git repo:\n    os.mkdir(path)\n    ok_(op.exists(path))\n    assert_raises(InvalidGitRepositoryError, GitRepo, path, create=False)\n    assert_false(op.exists(op.join(path, '.git')))\n\n    # 3. create=True, path doesn't exist:\n    gr = GitRepo(path2, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path2, '.git')))\n    ok_clean_git(path2, annex=False)\n\n    # 4. create=True, path exists, but no git repo:\n    gr = GitRepo(path, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n    ok_clean_git(path, annex=False)\n\n\n@with_tempfile\ndef test_GitRepo_init_options(path):\n    # passing an option, not explicitly defined in GitRepo class:\n    gr = GitRepo(path, create=True, bare=True)\n\n    cfg = gr.repo.config_reader()\n    ok_(cfg.get_value(section=\"core\", option=\"bare\"))\n\n\n@skip_if(external_versions['cmd:git'] < '2.14.0')\n@with_tree(\n    tree={\n        'subds': {\n            'file_name': ''\n        }\n    }\n)\ndef test_init_fail_under_known_subdir(path):\n    repo = GitRepo(path, create=True)\n    repo.add(op.join('subds', 'file_name'))\n    # Should fail even if we do not commit but only add to index:\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n    assert_in(\"file_name\", str(cme.exception))  # we provide a list of offenders\n    # and after we commit - the same story\n    repo.commit(\"added file\")\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n\n    # But it would succeed if we disable the checks\n    GitRepo(op.join(path, 'subds'), create=True, create_sanity_checks=False)\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_equals(path1, path2):\n\n    repo1 = GitRepo(path1)\n    repo2 = GitRepo(path1)\n    ok_(repo1 == repo2)\n    eq_(repo1, repo2)\n    repo2 = GitRepo(path2)\n    neq_(repo1, repo2)\n    ok_(repo1 != repo2)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:515\n@known_failure_windows\n@assert_cwd_unchanged\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_add(src, path):\n\n    gr = GitRepo.clone(src, path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n    added = gr.add(filename)\n\n    eq_(added, {'success': True, 'file': filename})\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    # uncommitted:\n    ok_(gr.dirty)\n\n    filename = \"another.txt\"\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"Another file to add to git\")\n\n    # include committing:\n    added2 = gr.add(filename)\n    gr.commit(msg=\"Add two files.\")\n    eq_(added2, {'success': True, 'file': filename})\n\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    ok_clean_git(path)\n\n\n@assert_cwd_unchanged\n@with_tree(tree={\n    'd': {'f1': 'content1',\n          'f2': 'content2'},\n    'file': 'content3',\n    'd2': {'f1': 'content1',\n          'f2': 'content2'},\n    'file2': 'content3'\n\n    })\ndef test_GitRepo_remove(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add('*')\n    gr.commit(\"committing all the files\")\n\n    eq_(gr.remove('file'), ['file'])\n    eq_(set(gr.remove('d', r=True, f=True)), {'d/f1', 'd/f2'})\n\n    eq_(set(gr.remove('*', r=True, f=True)), {'file2', 'd2/f1', 'd2/f2'})\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_GitRepo_commit(path):\n\n    gr = GitRepo(path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n\n    gr.add(filename)\n    gr.commit(\"Testing GitRepo.commit().\")\n    ok_clean_git(gr)\n    eq_(\"Testing GitRepo.commit().\",\n        gr.format_commit(\"%B\").strip())\n\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"changed content\")\n\n    gr.add(filename)\n    gr.commit(\"commit with options\", options=to_options(dry_run=True))\n    # wasn't actually committed:\n    ok_(gr.dirty)\n\n    # commit with empty message:\n    gr.commit()\n    ok_clean_git(gr)\n\n    # nothing to commit doesn't raise by default:\n    gr.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, gr.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(op.join(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"not-existing\")\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_indexed_files(src, path):\n\n    gr = GitRepo.clone(src, path)\n    idx_list = gr.get_indexed_files()\n\n    runner = Runner()\n    out = runner(['git', 'ls-files'], cwd=path)\n    out_list = list(filter(bool, out[0].split('\\n')))\n\n    for item in idx_list:\n        assert_in(item, out_list, \"%s not found in output of git ls-files in %s\" % (item, path))\n    for item in out_list:\n        assert_in(item, idx_list, \"%s not found in output of get_indexed_files in %s\" % (item, path))\n\n\n@with_tree([\n    ('empty', ''),\n    ('d1', (\n        ('empty', ''),\n        ('d2',\n            (('empty', ''),\n             )),\n        )),\n    ])\n@assert_cwd_unchanged(ok_to_chdir=True)\ndef test_normalize_path(git_path):\n\n    gr = GitRepo(git_path)\n\n    # cwd is currently outside the repo, so any relative path\n    # should be interpreted as relative to `annex_path`\n    assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, getpwd())\n\n    result = _normalize_path(gr.path, \"testfile\")\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # result = _normalize_path(gr.path, op.join('.', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    #\n    # result = _normalize_path(gr.path, op.join('testdir', '..', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    # Note: By now, normpath within normalize_paths() is disabled, therefore\n    # disable these tests.\n\n    result = _normalize_path(gr.path, op.join('testdir', 'testfile'))\n    eq_(result, op.join(\"testdir\", \"testfile\"), \"_normalize_path() returned %s\" % result)\n\n    result = _normalize_path(gr.path, op.join(git_path, \"testfile\"))\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # now we are inside, so\n    # OLD PHILOSOPHY: relative paths are relative to cwd and have\n    # to be converted to be relative to annex_path\n    # NEW PHILOSOPHY: still relative to repo! unless starts with . (curdir) or .. (pardir)\n    with chpwd(op.join(git_path, 'd1', 'd2')):\n\n        result = _normalize_path(gr.path, \"testfile\")\n        eq_(result, 'testfile', \"_normalize_path() returned %s\" % result)\n\n        # if not joined as directory name but just a prefix to the filename, should\n        # behave correctly\n        for d in (op.curdir, op.pardir):\n            result = _normalize_path(gr.path, d + \"testfile\")\n            eq_(result, d + 'testfile', \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.curdir, \"testfile\"))\n        eq_(result, op.join('d1', 'd2', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.pardir, 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, op.join(git_path, '..', 'outside'))\n\n        result = _normalize_path(gr.path, op.join(git_path, 'd1', 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n\ndef test_GitRepo_files_decorator():\n\n    class testclass(object):\n        def __init__(self):\n            self.path = op.join('some', 'where')\n\n        # TODO\n        # yoh:  logic is alien to me below why to have two since both look identical!\n        @normalize_paths\n        def decorated_many(self, files):\n            return files\n\n        @normalize_paths\n        def decorated_one(self, file_):\n            return file_\n\n    test_instance = testclass()\n\n    # When a single file passed -- single path returned\n    obscure_filename = get_most_obscure_supported_name()\n    file_to_test = op.join(test_instance.path, 'deep', obscure_filename)\n    # file doesn't exist\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = obscure_filename\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n\n    file_to_test = op.join(obscure_filename, 'beyond', 'obscure')\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = op.join(getpwd(), 'somewhere', 'else', obscure_filename)\n    assert_raises(FileNotInRepositoryError, test_instance.decorated_many,\n                  file_to_test)\n\n    # If a list passed -- list returned\n    files_to_test = ['now', op.join('a list', 'of'), 'paths']\n    expect = []\n    for item in files_to_test:\n        expect.append(_normalize_path(test_instance.path, item))\n    eq_(test_instance.decorated_many(files_to_test), expect)\n\n    eq_(test_instance.decorated_many(''), [])\n\n    assert_raises(ValueError, test_instance.decorated_many, 1)\n    assert_raises(ValueError, test_instance.decorated_one, 1)\n\n\n@skip_if_no_network\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_add(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    eq_(len(out), 1)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    assert_in('github', out)\n    eq_(len(out), 2)\n    eq_('git://github.com/datalad/testrepo--basic--r1', gr.config['remote.github.url'])\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_remove(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    gr.remove_remote('github')\n    out = gr.get_remotes()\n    eq_(len(out), 1)\n    assert_in('origin', out)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:491\n@known_failure_windows\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_remote_url(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    eq_(gr.get_remote_url('origin'), orig_path)\n    eq_(gr.get_remote_url('github'),\n                 'git://github.com/datalad/testrepo--basic--r1')\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_pull(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n    clone.pull()\n    ok_(op.exists(op.join(clone_path, filename)))\n\n    # While at it, let's test _get_remotes_having_commit a bit\n    from datalad.distribution.get import _get_remotes_having_commit\n    clone.add_remote(\"very_origin\", test_path)\n    clone.fetch(\"very_origin\")\n    eq_(\n        _get_remotes_having_commit(clone, clone.get_hexsha()),\n        ['origin']\n    )\n    prev_commit = clone.get_hexsha('HEAD^')\n    eq_(\n        set(_get_remotes_having_commit(clone, prev_commit)),\n        {'origin', 'very_origin'}\n    )\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_fetch(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    origin.checkout(\"new_branch\", ['-b'])\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n\n    fetched = clone.fetch(remote='origin')\n    # test FetchInfo list returned by fetch\n    eq_([u'origin/' + clone.get_active_branch(), u'origin/new_branch'],\n        [commit['ref'] for commit in fetched])\n\n    ok_clean_git(clone.path, annex=False)\n    assert_in(\"origin/new_branch\", clone.get_remote_branches())\n    assert_in(filename, clone.get_files(\"origin/new_branch\"))\n    assert_false(op.exists(op.join(clone_path, filename)))  # not checked out\n\n    # create a remote without an URL:\n    origin.add_remote('not-available', 'git://example.com/not/existing')\n    origin.config.unset('remote.not-available.url', where='local')\n\n    # fetch without provided URL\n    assert_raises(CommandError, origin.fetch, 'not-available')\n\n\ndef _path2localsshurl(path):\n    \"\"\"Helper to build valid localhost SSH urls on Windows too\"\"\"\n    from pathlib import Path\n    path = op.abspath(path)\n    p = Path(path)\n    if p.drive:\n        path = '/'.join(('/{}'.format(p.drive[0]),) + p.parts[1:])\n    url = \"ssh://localhost{}\".format(path)\n    return url\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_testrepos('.*basic.*', flavors=['local'])\n@with_tempfile\ndef test_GitRepo_ssh_fetch(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=False)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # we don't know any branches of the remote:\n    eq_([], repo.get_remote_branches())\n\n    fetched = repo.fetch(remote=\"ssh-remote\")\n    assert_in('ssh-remote/master', [commit['ref'] for commit in fetched])\n    ok_clean_git(repo)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually fetched it:\n    assert_in('ssh-remote/master', repo.get_remote_branches())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_pull(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify remote:\n    remote_repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(remote_repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    remote_repo.add(\"ssh_testfile.dat\")\n    remote_repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not locally known yet:\n    assert_not_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n    # pull changes:\n    repo.pull(remote=\"ssh-remote\", refspec=remote_repo.get_active_branch())\n    ok_clean_git(repo.path, annex=False)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually pulled the changes\n    assert_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_push(repo_path, remote_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify local repo:\n    repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    repo.add(\"ssh_testfile.dat\")\n    repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not known to the remote yet:\n    assert_not_in(\"ssh_testfile.dat\", remote_repo.get_indexed_files())\n\n    # push changes:\n    pushed = list(repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\"))\n    # test PushInfo\n    assert_in(\"refs/heads/ssh-test\", [p['from_ref'] for p in pushed])\n    assert_in(\"refs/heads/ssh-test\", [p['to_ref'] for p in pushed])\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # remote now knows the changes:\n    assert_in(\"ssh-test\", remote_repo.get_branches())\n    assert_in(\"ssh_testfile.dat\", remote_repo.get_files(\"ssh-test\"))\n\n    # amend to make it require \"--force\":\n    repo.commit(\"amended\", options=['--amend'])\n    # push without --force should yield an error:\n    res = repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\")\n    assert_in_results(\n        res,\n        from_ref='refs/heads/ssh-test',\n        to_ref='refs/heads/ssh-test',\n        operations=['rejected', 'error'],\n        note='[rejected] (non-fast-forward)',\n        remote='ssh-remote',\n    )\n    # now push using force:\n    repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\", force=True)\n    # correct commit message in remote:\n    assert_in(\"amended\",\n              remote_repo.format_commit(\n                  '%s',\n                  list(remote_repo.get_branch_commits_('ssh-test'))[-1]\n              ))\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_push_n_checkout(orig_path, clone_path):\n\n    origin = GitRepo(orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(clone_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    clone.add(filename)\n    clone.commit(\"new file added.\")\n    # TODO: need checkout first:\n    clone.push('origin', '+master:new-branch')\n    origin.checkout('new-branch')\n    ok_(op.exists(op.join(orig_path, filename)))\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_remote_update(path1, path2, path3):\n\n    git1 = GitRepo(path1)\n    git2 = GitRepo(path2)\n    git3 = GitRepo(path3)\n\n    git1.add_remote('git2', path2)\n    git1.add_remote('git3', path3)\n\n    # Setting up remote 'git2'\n    with open(op.join(path2, 'masterfile'), 'w') as f:\n        f.write(\"git2 in master\")\n    git2.add('masterfile')\n    git2.commit(\"Add something to master.\")\n    git2.checkout('branch2', ['-b'])\n    with open(op.join(path2, 'branch2file'), 'w') as f:\n        f.write(\"git2 in branch2\")\n    git2.add('branch2file')\n    git2.commit(\"Add something to branch2.\")\n\n    # Setting up remote 'git3'\n    with open(op.join(path3, 'masterfile'), 'w') as f:\n        f.write(\"git3 in master\")\n    git3.add('masterfile')\n    git3.commit(\"Add something to master.\")\n    git3.checkout('branch3', ['-b'])\n    with open(op.join(path3, 'branch3file'), 'w') as f:\n        f.write(\"git3 in branch3\")\n    git3.add('branch3file')\n    git3.commit(\"Add something to branch3.\")\n\n    git1.update_remote()\n\n    # checkouts are 'tests' themselves, since they'll raise CommandError\n    # if something went wrong\n    git1.checkout('branch2')\n    git1.checkout('branch3')\n\n    branches1 = git1.get_branches()\n    eq_({'branch2', 'branch3'}, set(branches1))\n\n\n# TODO: Why was it \"flavors=local_testrepo_flavors\" ? What's the windows issue here?\n@with_testrepos('.*git.*', flavors=['clone'])\n@with_tempfile\ndef test_GitRepo_get_files(url, path):\n\n    gr = GitRepo.clone(url, path)\n\n    # get the expected files via os for comparison:\n    os_files = set()\n    for (dirpath, dirnames, filenames) in os.walk(path):\n        rel_dir = os.path.relpath(dirpath, start=path)\n        if rel_dir.startswith(\".git\"):\n            continue\n        for file_ in filenames:\n            file_path = os.path.normpath(op.join(rel_dir, file_))\n            os_files.add(file_path)\n\n    # get the files via GitRepo:\n    local_files = set(gr.get_files())\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n\n    eq_(local_files, set(gr.get_indexed_files()))\n    eq_(local_files, remote_files)\n    eq_(local_files, os_files)\n\n    # create a different branch:\n    gr.checkout('new_branch', ['-b'])\n    filename = 'another_file.dat'\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"something\")\n    gr.add(filename)\n    gr.commit(\"Added.\")\n\n    # now get the files again:\n    local_files = set(gr.get_files())\n    eq_(local_files, os_files.union({filename}))\n    # retrieve remote branch again, which should not have changed:\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n    eq_(remote_files, os_files)\n    eq_(set([filename]), local_files.difference(remote_files))\n\n    # switch back and query non-active branch:\n    gr.checkout('master')\n    local_files = set(gr.get_files())\n    branch_files = set(gr.get_files(branch=\"new_branch\"))\n    eq_(set([filename]), branch_files.difference(local_files))\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:505\n@known_failure_windows\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_GitRepo_get_toppath(repo, tempdir, repo2):\n    reporeal = op.realpath(repo)\n    eq_(GitRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(GitRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    GitRepo(repo2, create=True)\n    repo2real = op.realpath(repo2)\n    nested = op.join(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(GitRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(GitRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(GitRepo.get_toppath(tempdir), None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_dirty(path):\n\n    repo = GitRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # untracked file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt')\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(not repo.dirty)\n    # modified file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt')\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # An empty directory doesn't count as dirty.\n    os.mkdir(op.join(path, \"empty\"))\n    ok_(not repo.dirty)\n    # Neither does an empty directory with an otherwise empty directory.\n    os.mkdir(op.join(path, \"empty\", \"empty-again\"))\n    ok_(not repo.dirty)\n\n    subm = GitRepo(repo.pathobj / \"subm\", create=True)\n    (subm.pathobj / \"subfile\").write_text(u\"\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n    (subm.pathobj / \"subfile\").write_text(u\"changed\")\n    ok_(repo.dirty)\n\n    # User configuration doesn't affect .dirty's answer.\n    repo.config.set(\"diff.ignoreSubmodules\", \"all\", where=\"local\")\n    ok_(repo.dirty)\n    # GitRepo.commit currently can't handle this setting, so remove it for the\n    # save() calls below.\n    repo.config.unset(\"diff.ignoreSubmodules\", where=\"local\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n\n    repo.config.set(\"status.showUntrackedFiles\", \"no\", where=\"local\")\n    create_tree(repo.path, {\"untracked_dir\": {\"a\": \"a\"}})\n    ok_(repo.dirty)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_merge_base(src):\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    assert_raises(ValueError, repo.get_merge_base, [])\n    branch1 = repo.get_active_branch()\n    branch1_hexsha = repo.get_hexsha()\n    eq_(len(branch1_hexsha), 40)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # Let's create a detached branch\n    branch2 = \"_detach_\"\n    repo.checkout(branch2, options=[\"--orphan\"])\n    # it will have all the files\n    # Must not do:  https://github.com/gitpython-developers/GitPython/issues/375\n    # repo.git_add('.')\n    repo.add('*')\n    # NOTE: fun part is that we should have at least a different commit message\n    # so it results in a different checksum ;)\n    repo.commit(\"committing again\")\n    assert(repo.get_indexed_files())  # we did commit\n    assert(repo.get_merge_base(branch1) is None)\n    assert(repo.get_merge_base([branch2, branch1]) is None)\n\n    # Let's merge them up -- then merge base should match the master\n    repo.merge(branch1, allow_unrelated=True)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # if points to some empty/non-existing branch - should also be None\n    assert(repo.get_merge_base(['nonexistent', branch2]) is None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_git_get_branch_commits_(src):\n\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    commits_default = list(repo.get_branch_commits_())\n    commits = list(repo.get_branch_commits_('master'))\n    eq_(commits, commits_default)\n    eq_(len(commits), 1)\n\n\n@with_testrepos(flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_get_tracking_branch(o_path, c_path):\n\n    clone = GitRepo.clone(o_path, c_path)\n    # Note, that the default branch might differ even if it is always 'master'.\n    # For direct mode annex repositories it would then be \"annex/direct/master\"\n    # for example. Therefore use whatever branch is checked out by default:\n    master_branch = clone.get_active_branch()\n    ok_(master_branch)\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n\n    clone.checkout('new_branch', ['-b'])\n\n    eq_((None, None), clone.get_tracking_branch())\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch(master_branch))\n\n    clone.checkout(master_branch, options=[\"--track\", \"-btopic\"])\n    eq_(('.', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n    eq_((None, None),\n        clone.get_tracking_branch(remote_only=True))\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_submodule_deinit(path):\n    from datalad.support.annexrepo import AnnexRepo\n\n    top_repo = AnnexRepo(path, create=False)\n    eq_({'subm 1', '2'}, {s.name for s in top_repo.get_submodules()})\n    # note: here init=True is ok, since we are using it just for testing\n    with swallow_logs(new_level=logging.WARN) as cml:\n        top_repo.update_submodule('subm 1', init=True)\n        assert_in('Do not use update_submodule with init=True', cml.out)\n    top_repo.update_submodule('2', init=True)\n\n    # ok_(all([s.module_exists() for s in top_repo.get_submodules()]))\n    # TODO: old assertion above if non-bare? (can't use \"direct mode\" in test_gitrepo)\n    # Alternatively: New testrepo (plain git submodules) and have a dedicated\n    # test for annexes in addition\n    ok_(all([GitRepo.is_valid_repo(op.join(top_repo.path, s.path))\n             for s in top_repo.get_submodules()]))\n\n    # modify submodule:\n    with open(op.join(top_repo.path, 'subm 1', 'file_ut.dat'), \"w\") as f:\n        f.write(\"some content\")\n\n    assert_raises(CommandError, top_repo.deinit_submodule, 'sub1')\n\n    # using force should work:\n    top_repo.deinit_submodule('subm 1', force=True)\n\n    ok_(not top_repo.repo.submodule('subm 1').module_exists())\n\n\n@with_testrepos(\".*basic_git.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_GitRepo_add_submodule(source, path):\n\n    top_repo = GitRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n    ok_clean_git(path)\n    ok_clean_git(op.join(path, 'sub'))\n\n\ndef test_GitRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef check_update_submodule_init_adjust_branch(is_ancestor, path):\n    src = GitRepo(op.join(path, \"src\"), create=True)\n    src_sub = GitRepo(op.join(src.path, \"sub\"), create=True)\n    src_sub.commit(msg=\"c0\", options=[\"--allow-empty\"])\n    src_sub.commit(msg=\"c1\", options=[\"--allow-empty\"])\n    src.add_submodule('sub', name='sub')\n    src.commit(msg=\"Add submodule\")\n\n    # Move subdataset past the registered commit...\n    hexsha_registered = src_sub.get_hexsha()\n    if is_ancestor:\n        # ... where the registered commit is an ancestor of the new one.\n        src_sub.commit(msg=\"c2\", options=[\"--allow-empty\"])\n    else:\n        # ... where the registered commit is NOT an ancestor of the new one.\n        src_sub.call_git([\"reset\", \"--hard\", \"master~1\"])  # c0\n    hexsha_sub = src_sub.get_hexsha()\n\n    clone = GitRepo.clone(url=src.path,\n                          path=op.join(path, \"clone\"),\n                          create=True)\n    clone_sub = GitRepo.clone(url=src_sub.path,\n                              path=op.join(clone.path, \"sub\"),\n                              create=True)\n    ok_(clone.dirty)\n    eq_(clone_sub.get_active_branch(), \"master\")\n    eq_(hexsha_sub, clone_sub.get_hexsha())\n\n    clone.update_submodule(\"sub\", init=True)\n\n    assert_false(clone.dirty)\n    eq_(hexsha_registered, clone_sub.get_hexsha())\n    if is_ancestor:\n        eq_(clone_sub.get_active_branch(), \"master\")\n    else:\n        assert_false(clone_sub.get_active_branch())\n\n\ndef test_GitRepo_update_submodule_init_adjust_branch():\n    yield check_update_submodule_init_adjust_branch, True\n    yield check_update_submodule_init_adjust_branch, False\n\n\ndef test_GitRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile\ndef test_get_submodules_parent_on_unborn_branch(path):\n    repo = GitRepo(path, create=True)\n    subrepo = GitRepo(op.join(path, \"sub\"), create=True)\n    subrepo.commit(msg=\"s\", options=[\"--allow-empty\"])\n    repo.add_submodule(path=\"sub\")\n    eq_([s.name for s in repo.get_submodules()],\n        [\"sub\"])\n\n\ndef test_to_options():\n\n    class Some(object):\n\n        def cmd_func(self, git_options=None, annex_options=None, options=None):\n\n            git_options = git_options[:] if git_options else []\n            annex_options = annex_options[:] if annex_options else []\n            options = options[:] if options else []\n\n            faked_cmd_call = ['git'] + git_options + ['annex'] + \\\n                             annex_options + ['my_cmd'] + options\n\n            return faked_cmd_call\n\n    eq_(Some().cmd_func(options=to_options(m=\"bla\", force=True)),\n        ['git', 'annex', 'my_cmd', '--force', '-m', 'bla'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\"),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C', '/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\", split_single_char_options=False),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n\n@with_tempfile\ndef test_GitRepo_count_objects(repo_path):\n\n    repo = GitRepo(repo_path, create=True)\n    # test if dictionary returned\n    eq_(isinstance(repo.count_objects, dict), True)\n    # test if dictionary contains keys and values we expect\n    empty_count = {'count': 0, 'garbage': 0,  'in-pack': 0, 'packs': 0, 'prune-packable': 0,\n                   'size': 0, 'size-garbage': 0, 'size-pack': 0}\n    eq_(empty_count, repo.count_objects)\n\n\n# this is simply broken on win, but less important\n# https://github.com/datalad/datalad/issues/3639\n@skip_if_on_windows\n@with_tempfile\ndef test_optimized_cloning(path):\n    # make test repo with one file and one commit\n    originpath = op.join(path, 'origin')\n    repo = GitRepo(originpath, create=True)\n    with open(op.join(originpath, 'test'), 'w') as f:\n        f.write('some')\n    repo.add('test')\n    repo.commit('init')\n    ok_clean_git(originpath, annex=False)\n    from glob import glob\n\n    def _get_inodes(repo):\n        return dict(\n            [(os.path.join(*o.split(os.sep)[-2:]),\n              os.stat(o).st_ino)\n             for o in glob(os.path.join(repo.path,\n                                        repo.get_git_dir(repo),\n                                        'objects', '*', '*'))])\n\n    origin_inodes = _get_inodes(repo)\n    # now clone it in different ways and see what happens to the object storage\n    from datalad.support.network import get_local_file_url\n    clonepath = op.join(path, 'clone')\n    for src in (originpath, get_local_file_url(originpath, compatibility='git')):\n        # deprecated\n        assert_raises(DeprecatedError, GitRepo, url=src, path=clonepath)\n        clone = GitRepo.clone(url=src, path=clonepath, create=True)\n        clone_inodes = _get_inodes(clone)\n        eq_(origin_inodes, clone_inodes, msg='with src={}'.format(src))\n        rmtree(clonepath)\n#        del clone\n#        gc.collect()\n        # Note: del needed, since otherwise WeakSingletonRepo would just\n        # return the original object in second run\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_gitpy_injection(path, path2):\n\n    gr = GitRepo(path, create=True)\n    gr._GIT_COMMON_OPTIONS.extend(['test-option'])\n\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # once set, these option should be persistent across git calls:\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.another_unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # but other repos should not be affected:\n    gr2 = GitRepo(path2, create=True)\n    with assert_raises(GitCommandError) as cme:\n        gr2.repo.git.unknown_git_command()\n    assert_not_in('test-option', exc_str(cme.exception))\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_GitRepo_flyweight(path1, path2):\n\n    repo1 = GitRepo(path1, create=True)\n    assert_is_instance(repo1, GitRepo)\n    # instantiate again:\n    repo2 = GitRepo(path1, create=False)\n    assert_is_instance(repo2, GitRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in a different way:\n    with chpwd(path1):\n        repo3 = GitRepo(op.relpath(path1, start=path2), create=False)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # and realpath attribute is the same, so they are still equal:\n    ok_(repo1 == repo3)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_GitRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(GitRepo, path, store)\n\n\n@with_tree(tree={'ignore-sub.me': {'a_file.txt': 'some content'},\n                 'ignore.me': 'ignored content',\n                 'dontigno.re': 'other content'})\ndef test_GitRepo_gitignore(path):\n\n    gr = GitRepo(path, create=True)\n    sub = GitRepo(op.join(path, 'ignore-sub.me'))\n    # we need to commit something, otherwise add_submodule\n    # will already refuse the submodule for having no commit\n    sub.add('a_file.txt')\n    sub.commit()\n\n    from ..exceptions import GitIgnoreError\n\n    with open(op.join(path, '.gitignore'), \"w\") as f:\n        f.write(\"*.me\")\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add('ignore.me')\n    eq_(cme.exception.paths, ['ignore.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add_submodule(path='ignore-sub.me')\n    eq_(cme.exception.paths, ['ignore-sub.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add(['ignore.me', 'dontigno.re', op.join('ignore-sub.me', 'a_file.txt')])\n    eq_(set(cme.exception.paths), {'ignore.me', 'ignore-sub.me'})\n\n    eq_(gr.get_gitattributes('.')['.'], {})  # nothing is recorded within .gitattributes\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_set_remote_url(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add_remote('some', 'http://example.com/.git')\n    eq_(gr.config['remote.some.url'],\n                 'http://example.com/.git')\n    # change url:\n    gr.set_remote_url('some', 'http://believe.it')\n    eq_(gr.config['remote.some.url'],\n                 'http://believe.it')\n\n    # set push url:\n    gr.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    eq_(gr.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n\n    # add remote without url\n    url2 = 'http://repo2.example.com/.git'\n    gr.add_remote('some-without-url', url2)\n    eq_(gr.config['remote.some-without-url.url'], url2)\n    # \"remove\" it\n    gr.config.unset('remote.some-without-url.url', where='local')\n    with assert_raises(KeyError):\n        gr.config['remote.some-without-url.url']\n    eq_(set(gr.get_remotes()), {'some', 'some-without-url'})\n    eq_(set(gr.get_remotes(with_urls_only=True)), {'some'})\n\n\n@with_tempfile(mkdir=True)\ndef test_gitattributes(path):\n    gr = GitRepo(path, create=True)\n    # starts without any attributes file\n    ok_(not op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {})\n    # bool is a tag or unsets, anything else is key/value\n    gr.set_gitattributes([('*', {'tag': True}), ('*', {'sec.key': 'val'})])\n    ok_(op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {'tag': True, 'sec.key': 'val'})\n    # unset by amending the record, but does not remove notion of the\n    # tag entirely\n    gr.set_gitattributes([('*', {'tag': False})])\n    eq_(gr.get_gitattributes('.')['.'], {'tag': False, 'sec.key': 'val'})\n    # attributes file is not added or commited, we can ignore such\n    # attributes\n    eq_(gr.get_gitattributes('.', index_only=True)['.'], {})\n\n    # we can send absolute path patterns and write to any file, and\n    # the patterns will be translated relative to the target file\n    gr.set_gitattributes([\n        (op.join(gr.path, 'relative', 'ikethemike/**'), {'bang': True})],\n        attrfile=op.join('relative', '.gitattributes'))\n    # directory and file get created\n    ok_(op.exists(op.join(gr.path, 'relative', '.gitattributes')))\n    eq_(gr.get_gitattributes(\n        op.join(gr.path, 'relative', 'ikethemike', 'probe')),\n        # always comes out relative to the repo root, even if abs goes in\n        {op.join('relative', 'ikethemike', 'probe'):\n            {'tag': False, 'sec.key': 'val', 'bang': True}})\n    if get_encoding_info()['default'] != 'ascii' and not on_windows:\n        # do not perform this on obscure systems without anything like UTF\n        # it is not relevant whether a path actually exists, and paths\n        # with spaces and other funky stuff are just fine\n        funky = u'{} {}'.format(\n            get_most_obscure_supported_name(),\n            get_most_obscure_supported_name())\n        gr.set_gitattributes([(funky, {'this': 'that'})])\n        eq_(gr.get_gitattributes(funky)[funky], {\n            'this': 'that',\n            'tag': False,\n            'sec.key': 'val',\n        })\n\n    # mode='w' should replace the entire file:\n    gr.set_gitattributes([('**', {'some': 'nonsense'})], mode='w')\n    eq_(gr.get_gitattributes('.')['.'], {'some': 'nonsense'})\n\n\n@with_tempfile(mkdir=True)\ndef test_get_hexsha_tag(path):\n    gr = GitRepo(path, create=True)\n    gr.commit(msg=\"msg\", options=[\"--allow-empty\"])\n    gr.tag(\"atag\", message=\"atag msg\")\n    # get_hexsha() dereferences a tag to a commit.\n    eq_(gr.get_hexsha(\"atag\"), gr.get_hexsha())\n\n\n@with_tempfile(mkdir=True)\ndef test_get_tags(path):\n    from unittest.mock import patch\n\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_tags(), [])\n    eq_(gr.describe(), None)\n\n    # Explicitly override the committer date because tests may set it to a\n    # fixed value, but we want to check that the returned tags are sorted by\n    # the date the tag (for annotaged tags) or commit (for lightweight tags)\n    # was created.\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Thu, 07 Apr 2005 22:13:13 +0200\"}):\n        create_tree(gr.path, {'file': \"\"})\n        gr.add('file')\n        gr.commit(msg=\"msg\")\n        eq_(gr.get_tags(), [])\n        eq_(gr.describe(), None)\n\n        gr.tag(\"nonannotated\")\n        tags1 = [{'name': 'nonannotated', 'hexsha': gr.get_hexsha()}]\n        eq_(gr.get_tags(), tags1)\n        eq_(gr.describe(), None)\n        eq_(gr.describe(tags=True), tags1[0]['name'])\n\n    first_commit = gr.get_hexsha()\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 08 Apr 2005 22:13:13 +0200\"}):\n\n        create_tree(gr.path, {'file': \"123\"})\n        gr.add('file')\n        gr.commit(msg=\"changed\")\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 09 Apr 2005 22:13:13 +0200\"}):\n        gr.tag(\"annotated\", message=\"annotation\")\n    # The annotated tag happened later, so it comes last.\n    tags2 = tags1 + [{'name': 'annotated', 'hexsha': gr.get_hexsha()}]\n    eq_(gr.get_tags(), tags2)\n    eq_(gr.describe(), tags2[1]['name'])\n\n    # compare prev commit\n    eq_(gr.describe(commitish=first_commit), None)\n    eq_(gr.describe(commitish=first_commit, tags=True), tags1[0]['name'])\n\n    gr.tag('specific', commit='HEAD~1')\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD~1'))\n    assert_in('specific', gr.get_tags(output='name'))\n\n    # retag a different commit\n    assert_raises(CommandError, gr.tag, 'specific', commit='HEAD')\n    # force it\n    gr.tag('specific', commit='HEAD', options=['-f'])\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD'))\n\n    # delete\n    gr.call_git(['tag', '-d', 'specific'])\n    eq_(gr.get_tags(), tags2)\n    # more than one\n    gr.tag('one')\n    gr.tag('two')\n    gr.call_git(['tag', '-d', 'one', 'two'])\n    eq_(gr.get_tags(), tags2)\n\n\n@with_tree(tree={'1': \"\"})\ndef test_get_commit_date(path):\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_commit_date(), None)\n\n    # Let's make a commit with a custom date\n    DATE = \"Wed Mar 14 03:47:30 2018 -0000\"\n    DATE_EPOCH = 1520999250\n    gr.add('1')\n    gr.commit(\"committed\", date=DATE)\n    gr = GitRepo(path, create=True)\n    date = gr.get_commit_date()\n    neq_(date, None)\n    eq_(date, DATE_EPOCH)\n\n    eq_(date, gr.get_commit_date('master'))\n    # and even if we get into a detached head\n    gr.checkout(gr.get_hexsha())\n    eq_(gr.get_active_branch(), None)\n    eq_(date, gr.get_commit_date('master'))\n\n\n@with_tree(tree={\"foo\": \"foo content\",\n                 \"bar\": \"bar content\"})\ndef test_fake_dates(path):\n    gr = GitRepo(path, create=True, fake_dates=True)\n\n    gr.add(\"foo\")\n    gr.commit(\"commit foo\")\n\n    seconds_initial = gr.config.obtain(\"datalad.fake-dates-start\")\n\n    # First commit is incremented by 1 second.\n    eq_(seconds_initial + 1, gr.get_commit_date())\n\n    # The second commit by 2.\n    gr.add(\"bar\")\n    gr.commit(\"commit bar\")\n    eq_(seconds_initial + 2, gr.get_commit_date())\n\n    # If we checkout another branch, its time is still based on the latest\n    # timestamp in any local branch.\n    gr.checkout(\"other\", options=[\"--orphan\"])\n    with open(op.join(path, \"baz\"), \"w\") as ofh:\n        ofh.write(\"baz content\")\n    gr.add(\"baz\")\n    gr.commit(\"commit baz\")\n    eq_(gr.get_active_branch(), \"other\")\n    eq_(seconds_initial + 3, gr.get_commit_date())\n\n\ndef test_guard_BadName():\n    from gitdb.exc import BadName\n\n    calls = []\n\n    class Vulnerable(object):\n        def precommit(self):\n            calls.append('precommit')\n\n        @guard_BadName\n        def __call__(self, x, y=2):\n            if not calls:\n                calls.append(1)\n                raise BadName\n            return x+y\n    v = Vulnerable()\n    eq_(v(1, y=3), 4)\n    eq_(calls, [1, 'precommit'])\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\ndef test_custom_runner_protocol(path):\n    # Check that a runner with a non-default protocol gets wired up correctly.\n    prot = ExecutionTimeProtocol()\n    gr = GitRepo(path, runner=Runner(cwd=path, protocol=prot), create=True)\n\n    ok_(len(prot) > 0)\n    ok_(prot[0]['duration'] >= 0)\n\n    def check(prev_len, prot, command):\n        # Check that the list grew and has the expected command without\n        # assuming that it gained _only_ a one command.\n        ok_(len(prot) > prev_len)\n        assert_in(command,\n                  sum([p[\"command\"] for p in prot[prev_len:]], []))\n\n    prev_len = len(prot)\n    gr.add(\"foo\")\n    check(prev_len, prot, \"add\")\n\n    # commit no longer uses a Runner with protocol capabilities\n    #prev_len = len(prot)\n    #gr.commit(\"commit foo\")\n    #check(prev_len, prot, \"commit\")\n\n    ok_(all(p['duration'] >= 0 for p in prot))\n\n\n@with_tempfile(mkdir=True)\ndef test_duecredit(path):\n    # Just to check that no obvious side-effects\n    run = Runner(cwd=path).run\n    cmd = [\n        sys.executable, \"-c\",\n        \"from datalad.support.gitrepo import GitRepo; GitRepo(%r, create=True)\" % path\n    ]\n\n    env = os.environ.copy()\n\n    # Test with duecredit not enabled for sure\n    env.pop('DUECREDIT_ENABLE', None)\n    # Alternative workaround for what to be fixed by\n    # https://github.com/datalad/datalad/pull/3215\n    # where underlying datalad process might issue a warning since our temp\n    # cwd is not matching possibly present PWD env variable\n    env.pop('PWD', None)\n\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err  # Let's not depend on where duecredit decides to spit out\n    # All quiet\n    eq_(outs, '')\n\n    # and now enable DUECREDIT - output could come to stderr\n    env['DUECREDIT_ENABLE'] = '1'\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err\n\n    if external_versions['duecredit']:\n        assert_in('Data management and distribution platform', outs)\n    else:\n        eq_(outs, '')\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_revisions(path):\n    gr = GitRepo(path, create=True)\n\n    def commit(msg):\n        gr.commit(msg=msg, options=[\"--allow-empty\"])\n\n    # We catch the error and return empty if the current branch doesn't have a\n    # commit checked out.\n    eq_(gr.get_revisions(), [])\n\n    # But will raise if on a bad ref name, including an unborn branch.\n    with assert_raises(CommandError):\n        gr.get_revisions(\"master\")\n\n    # By default, we query HEAD.\n    commit(\"1\")\n    eq_(len(gr.get_revisions()), 1)\n\n    gr.checkout(\"other\", options=[\"-b\"])\n    commit(\"2\")\n\n    # We can also query branch by name.\n    eq_(len(gr.get_revisions(\"master\")), 1)\n    eq_(len(gr.get_revisions(\"other\")), 2)\n\n    # \"name\" is sugar for [\"name\"].\n    eq_(gr.get_revisions(\"master\"),\n        gr.get_revisions([\"master\"]))\n\n    gr.checkout(\"master\")\n    commit(\"3\")\n    eq_(len(gr.get_revisions(\"master\")), 2)\n    # We can pass multiple revisions...\n    eq_(len(gr.get_revisions([\"master\", \"other\"])), 3)\n    # ... or options like --all and --branches\n    eq_(gr.get_revisions([\"master\", \"other\"]),\n        gr.get_revisions(options=[\"--all\"]))\n\n    # Ranges are supported.\n    eq_(gr.get_revisions(\"master..\"), [])\n\n\n@with_tree({\"foo\": \"foo\"})\ndef test_gitrepo_add_to_git_with_annex_v7(path):\n    from datalad.support.annexrepo import AnnexRepo\n    ar = AnnexRepo(path, create=True, version=7)\n    gr = GitRepo(path)\n    gr.add(\"foo\")\n    gr.commit(msg=\"c1\")\n    assert_false(ar.is_under_annex(\"foo\"))\n\n\n@with_tree({\"foo\": \"foo\", \"bar\": \"bar\"})\ndef test_gitrepo_call_git_methods(path):\n    gr = GitRepo(path)\n    gr.add([\"foo\", \"bar\"])\n    gr.commit(msg=\"foobar\")\n    gr.call_git([\"mv\"], files=[\"foo\", \"foo.txt\"])\n    ok_(op.exists(op.join(gr.path, 'foo.txt')))\n\n    for expect_fail, check in [(False, assert_in),\n                               (True, assert_not_in)]:\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            with assert_raises(CommandError):\n                gr.call_git([\"mv\"], files=[\"notthere\", \"dest\"],\n                            expect_fail=expect_fail)\n            check(\"notthere\", cml.out)\n\n    eq_(list(gr.call_git_items_([\"ls-files\"])),\n        [\"bar\", \"foo.txt\"])\n    eq_(list(gr.call_git_items_([\"ls-files\", \"-z\"], sep=\"\\0\")),\n        # Note: The custom separator has trailing empty item, but this is an\n        # arbitrary command with unknown output it isn't safe to trim it.\n        [\"bar\", \"foo.txt\", \"\"])\n\n    with assert_raises(AssertionError):\n        gr.call_git_oneline([\"ls-files\"])\n\n    eq_(gr.call_git_oneline([\"ls-files\"], files=[\"bar\"]),\n        \"bar\")\n\n    ok_(gr.call_git_success([\"rev-parse\", \"HEAD^{commit}\"]))\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        assert_false(gr.call_git_success([\"rev-parse\", \"HEAD^{blob}\"]))\n        assert_not_in(\"blob\", cml.out)\n\n\n@skip_if_no_network\n@with_tempfile\ndef _test_protocols(proto, destdir):\n    GitRepo.clone('%s://github.com/datalad-tester/testtt' % proto, destdir)\n\n\n@integration\ndef test_protocols():\n    # git-annex-standalone build can get git bundle which would fail to\n    # download via https, resulting in messages such as\n    #  fatal: unable to find remote helper for 'https'\n    # which happened with git-annex-standalone 7.20191017+git2-g7b13db551-1~ndall+1\n\n    # http is well tested already\n    for proto in 'git', 'https':\n        yield _test_protocols, proto\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class GitRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import assert_is_instance\n\nimport logging\n\nimport os\nimport os.path as op\n\nimport sys\n\n\nfrom datalad import get_encoding_info\nfrom datalad.cmd import Runner\n\nfrom datalad.utils import (\n    chpwd,\n    getpwd,\n    on_windows,\n    rmtree,\n    unlink,\n)\nfrom datalad.tests.utils import (\n    assert_cwd_unchanged,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    create_tree,\n    eq_,\n    get_most_obscure_supported_name,\n    integration,\n    known_failure_windows,\n    local_testrepo_flavors,\n    neq_,\n    ok_,\n    ok_clean_git,\n    skip_if,\n    skip_if_no_network,\n    skip_if_on_windows,\n    skip_ssh,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_testrepos,\n    with_tree,\n)\nfrom datalad.tests.utils_testrepos import BasicAnnexTestRepo\n\nfrom datalad.dochelpers import exc_str\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.support.gitrepo import (\n    _normalize_path,\n    GitCommandError,\n    gitpy,\n    GitRepo,\n    guard_BadName,\n    InvalidGitRepositoryError,\n    normalize_paths,\n    NoSuchPathError,\n    to_options,\n)\nfrom datalad.support.exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    PathKnownToRepositoryError,\n)\nfrom datalad.support.external_versions import external_versions\nfrom datalad.support.protocol import ExecutionTimeProtocol\nfrom .utils import check_repo_deals_with_inode_change\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_invalid_path(path):\n    with chpwd(path):\n        assert_raises(ValueError, GitRepo, path=\"git://some/url\", create=True)\n        ok_(not op.exists(op.join(path, \"git:\")))\n        assert_raises(ValueError, GitRepo, path=\"file://some/relative/path\", create=True)\n        ok_(not op.exists(op.join(path, \"file:\")))\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_instance_from_clone(src, dst):\n\n    gr = GitRepo.clone(src, dst)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    assert_is_instance(gr.repo, gitpy.Repo,\n                       \"Failed to instantiate GitPython Repo object.\")\n    ok_(op.exists(op.join(dst, '.git')))\n\n    # do it again should raise GitCommandError since git will notice there's\n    # already a git-repo at that path and therefore can't clone to `dst`\n    # Note: Since GitRepo is now a WeakSingletonRepo, this is prevented from\n    # happening atm. Disabling for now:\n#    raise SkipTest(\"Disabled for RF: WeakSingletonRepo\")\n    with swallow_logs() as logs:\n        assert_raises(GitCommandError, GitRepo.clone, src, dst)\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\ndef test_GitRepo_instance_from_existing(path):\n\n    gr = GitRepo(path)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_instance_from_not_existing(path, path2):\n    # 1. create=False and path doesn't exist:\n    assert_raises(NoSuchPathError, GitRepo, path, create=False)\n    assert_false(op.exists(path))\n\n    # 2. create=False, path exists, but no git repo:\n    os.mkdir(path)\n    ok_(op.exists(path))\n    assert_raises(InvalidGitRepositoryError, GitRepo, path, create=False)\n    assert_false(op.exists(op.join(path, '.git')))\n\n    # 3. create=True, path doesn't exist:\n    gr = GitRepo(path2, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path2, '.git')))\n    ok_clean_git(path2, annex=False)\n\n    # 4. create=True, path exists, but no git repo:\n    gr = GitRepo(path, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n    ok_clean_git(path, annex=False)\n\n\n@with_tempfile\ndef test_GitRepo_init_options(path):\n    # passing an option, not explicitly defined in GitRepo class:\n    gr = GitRepo(path, create=True, bare=True)\n\n    cfg = gr.repo.config_reader()\n    ok_(cfg.get_value(section=\"core\", option=\"bare\"))\n\n\n@skip_if(external_versions['cmd:git'] < '2.14.0')\n@with_tree(\n    tree={\n        'subds': {\n            'file_name': ''\n        }\n    }\n)\ndef test_init_fail_under_known_subdir(path):\n    repo = GitRepo(path, create=True)\n    repo.add(op.join('subds', 'file_name'))\n    # Should fail even if we do not commit but only add to index:\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n    assert_in(\"file_name\", str(cme.exception))  # we provide a list of offenders\n    # and after we commit - the same story\n    repo.commit(\"added file\")\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n\n    # But it would succeed if we disable the checks\n    GitRepo(op.join(path, 'subds'), create=True, create_sanity_checks=False)\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_equals(path1, path2):\n\n    repo1 = GitRepo(path1)\n    repo2 = GitRepo(path1)\n    ok_(repo1 == repo2)\n    eq_(repo1, repo2)\n    repo2 = GitRepo(path2)\n    neq_(repo1, repo2)\n    ok_(repo1 != repo2)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:515\n@known_failure_windows\n@assert_cwd_unchanged\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_add(src, path):\n\n    gr = GitRepo.clone(src, path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n    added = gr.add(filename)\n\n    eq_(added, {'success': True, 'file': filename})\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    # uncommitted:\n    ok_(gr.dirty)\n\n    filename = \"another.txt\"\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"Another file to add to git\")\n\n    # include committing:\n    added2 = gr.add(filename)\n    gr.commit(msg=\"Add two files.\")\n    eq_(added2, {'success': True, 'file': filename})\n\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    ok_clean_git(path)\n\n\n@assert_cwd_unchanged\n@with_tree(tree={\n    'd': {'f1': 'content1',\n          'f2': 'content2'},\n    'file': 'content3',\n    'd2': {'f1': 'content1',\n          'f2': 'content2'},\n    'file2': 'content3'\n\n    })\ndef test_GitRepo_remove(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add('*')\n    gr.commit(\"committing all the files\")\n\n    eq_(gr.remove('file'), ['file'])\n    eq_(set(gr.remove('d', r=True, f=True)), {'d/f1', 'd/f2'})\n\n    eq_(set(gr.remove('*', r=True, f=True)), {'file2', 'd2/f1', 'd2/f2'})\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_GitRepo_commit(path):\n\n    gr = GitRepo(path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n\n    gr.add(filename)\n    gr.commit(\"Testing GitRepo.commit().\")\n    ok_clean_git(gr)\n    eq_(\"Testing GitRepo.commit().\",\n        gr.format_commit(\"%B\").strip())\n\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"changed content\")\n\n    gr.add(filename)\n    gr.commit(\"commit with options\", options=to_options(dry_run=True))\n    # wasn't actually committed:\n    ok_(gr.dirty)\n\n    # commit with empty message:\n    gr.commit()\n    ok_clean_git(gr)\n\n    # nothing to commit doesn't raise by default:\n    gr.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, gr.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(op.join(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"not-existing\")\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_indexed_files(src, path):\n\n    gr = GitRepo.clone(src, path)\n    idx_list = gr.get_indexed_files()\n\n    runner = Runner()\n    out = runner(['git', 'ls-files'], cwd=path)\n    out_list = list(filter(bool, out[0].split('\\n')))\n\n    for item in idx_list:\n        assert_in(item, out_list, \"%s not found in output of git ls-files in %s\" % (item, path))\n    for item in out_list:\n        assert_in(item, idx_list, \"%s not found in output of get_indexed_files in %s\" % (item, path))\n\n\n@with_tree([\n    ('empty', ''),\n    ('d1', (\n        ('empty', ''),\n        ('d2',\n            (('empty', ''),\n             )),\n        )),\n    ])\n@assert_cwd_unchanged(ok_to_chdir=True)\ndef test_normalize_path(git_path):\n\n    gr = GitRepo(git_path)\n\n    # cwd is currently outside the repo, so any relative path\n    # should be interpreted as relative to `annex_path`\n    assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, getpwd())\n\n    result = _normalize_path(gr.path, \"testfile\")\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # result = _normalize_path(gr.path, op.join('.', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    #\n    # result = _normalize_path(gr.path, op.join('testdir', '..', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    # Note: By now, normpath within normalize_paths() is disabled, therefore\n    # disable these tests.\n\n    result = _normalize_path(gr.path, op.join('testdir', 'testfile'))\n    eq_(result, op.join(\"testdir\", \"testfile\"), \"_normalize_path() returned %s\" % result)\n\n    result = _normalize_path(gr.path, op.join(git_path, \"testfile\"))\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # now we are inside, so\n    # OLD PHILOSOPHY: relative paths are relative to cwd and have\n    # to be converted to be relative to annex_path\n    # NEW PHILOSOPHY: still relative to repo! unless starts with . (curdir) or .. (pardir)\n    with chpwd(op.join(git_path, 'd1', 'd2')):\n\n        result = _normalize_path(gr.path, \"testfile\")\n        eq_(result, 'testfile', \"_normalize_path() returned %s\" % result)\n\n        # if not joined as directory name but just a prefix to the filename, should\n        # behave correctly\n        for d in (op.curdir, op.pardir):\n            result = _normalize_path(gr.path, d + \"testfile\")\n            eq_(result, d + 'testfile', \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.curdir, \"testfile\"))\n        eq_(result, op.join('d1', 'd2', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.pardir, 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, op.join(git_path, '..', 'outside'))\n\n        result = _normalize_path(gr.path, op.join(git_path, 'd1', 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n\ndef test_GitRepo_files_decorator():\n\n    class testclass(object):\n        def __init__(self):\n            self.path = op.join('some', 'where')\n\n        # TODO\n        # yoh:  logic is alien to me below why to have two since both look identical!\n        @normalize_paths\n        def decorated_many(self, files):\n            return files\n\n        @normalize_paths\n        def decorated_one(self, file_):\n            return file_\n\n    test_instance = testclass()\n\n    # When a single file passed -- single path returned\n    obscure_filename = get_most_obscure_supported_name()\n    file_to_test = op.join(test_instance.path, 'deep', obscure_filename)\n    # file doesn't exist\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = obscure_filename\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n\n    file_to_test = op.join(obscure_filename, 'beyond', 'obscure')\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = op.join(getpwd(), 'somewhere', 'else', obscure_filename)\n    assert_raises(FileNotInRepositoryError, test_instance.decorated_many,\n                  file_to_test)\n\n    # If a list passed -- list returned\n    files_to_test = ['now', op.join('a list', 'of'), 'paths']\n    expect = []\n    for item in files_to_test:\n        expect.append(_normalize_path(test_instance.path, item))\n    eq_(test_instance.decorated_many(files_to_test), expect)\n\n    eq_(test_instance.decorated_many(''), [])\n\n    assert_raises(ValueError, test_instance.decorated_many, 1)\n    assert_raises(ValueError, test_instance.decorated_one, 1)\n\n\n@skip_if_no_network\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_add(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    eq_(len(out), 1)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    assert_in('github', out)\n    eq_(len(out), 2)\n    eq_('git://github.com/datalad/testrepo--basic--r1', gr.config['remote.github.url'])\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_remove(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    gr.remove_remote('github')\n    out = gr.get_remotes()\n    eq_(len(out), 1)\n    assert_in('origin', out)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:491\n@known_failure_windows\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_remote_url(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    eq_(gr.get_remote_url('origin'), orig_path)\n    eq_(gr.get_remote_url('github'),\n                 'git://github.com/datalad/testrepo--basic--r1')\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_pull(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n    clone.pull()\n    ok_(op.exists(op.join(clone_path, filename)))\n\n    # While at it, let's test _get_remotes_having_commit a bit\n    from datalad.distribution.get import _get_remotes_having_commit\n    clone.add_remote(\"very_origin\", test_path)\n    clone.fetch(\"very_origin\")\n    eq_(\n        _get_remotes_having_commit(clone, clone.get_hexsha()),\n        ['origin']\n    )\n    prev_commit = clone.get_hexsha('HEAD^')\n    eq_(\n        set(_get_remotes_having_commit(clone, prev_commit)),\n        {'origin', 'very_origin'}\n    )\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_fetch(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    origin.checkout(\"new_branch\", ['-b'])\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n\n    fetched = clone.fetch(remote='origin')\n    # test FetchInfo list returned by fetch\n    eq_([u'origin/' + clone.get_active_branch(), u'origin/new_branch'],\n        [commit['ref'] for commit in fetched])\n\n    ok_clean_git(clone.path, annex=False)\n    assert_in(\"origin/new_branch\", clone.get_remote_branches())\n    assert_in(filename, clone.get_files(\"origin/new_branch\"))\n    assert_false(op.exists(op.join(clone_path, filename)))  # not checked out\n\n    # create a remote without an URL:\n    origin.add_remote('not-available', 'git://example.com/not/existing')\n    origin.config.unset('remote.not-available.url', where='local')\n\n    # fetch without provided URL\n    assert_raises(CommandError, origin.fetch, 'not-available')\n\n\ndef _path2localsshurl(path):\n    \"\"\"Helper to build valid localhost SSH urls on Windows too\"\"\"\n    from pathlib import Path\n    path = op.abspath(path)\n    p = Path(path)\n    if p.drive:\n        path = '/'.join(('/{}'.format(p.drive[0]),) + p.parts[1:])\n    url = \"ssh://localhost{}\".format(path)\n    return url\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_testrepos('.*basic.*', flavors=['local'])\n@with_tempfile\ndef test_GitRepo_ssh_fetch(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=False)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # we don't know any branches of the remote:\n    eq_([], repo.get_remote_branches())\n\n    fetched = repo.fetch(remote=\"ssh-remote\")\n    assert_in('ssh-remote/master', [commit['ref'] for commit in fetched])\n    ok_clean_git(repo)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually fetched it:\n    assert_in('ssh-remote/master', repo.get_remote_branches())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_pull(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify remote:\n    remote_repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(remote_repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    remote_repo.add(\"ssh_testfile.dat\")\n    remote_repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not locally known yet:\n    assert_not_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n    # pull changes:\n    repo.pull(remote=\"ssh-remote\", refspec=remote_repo.get_active_branch())\n    ok_clean_git(repo.path, annex=False)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually pulled the changes\n    assert_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_push(repo_path, remote_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify local repo:\n    repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    repo.add(\"ssh_testfile.dat\")\n    repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not known to the remote yet:\n    assert_not_in(\"ssh_testfile.dat\", remote_repo.get_indexed_files())\n\n    # push changes:\n    pushed = list(repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\"))\n    # test PushInfo\n    assert_in(\"refs/heads/ssh-test\", [p['from_ref'] for p in pushed])\n    assert_in(\"refs/heads/ssh-test\", [p['to_ref'] for p in pushed])\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # remote now knows the changes:\n    assert_in(\"ssh-test\", remote_repo.get_branches())\n    assert_in(\"ssh_testfile.dat\", remote_repo.get_files(\"ssh-test\"))\n\n    # amend to make it require \"--force\":\n    repo.commit(\"amended\", options=['--amend'])\n    # push without --force should yield an error:\n    res = repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\")\n    assert_in_results(\n        res,\n        from_ref='refs/heads/ssh-test',\n        to_ref='refs/heads/ssh-test',\n        operations=['rejected', 'error'],\n        note='[rejected] (non-fast-forward)',\n        remote='ssh-remote',\n    )\n    # now push using force:\n    repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\", force=True)\n    # correct commit message in remote:\n    assert_in(\"amended\",\n              remote_repo.format_commit(\n                  '%s',\n                  list(remote_repo.get_branch_commits_('ssh-test'))[-1]\n              ))\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_push_n_checkout(orig_path, clone_path):\n\n    origin = GitRepo(orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(clone_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    clone.add(filename)\n    clone.commit(\"new file added.\")\n    # TODO: need checkout first:\n    clone.push('origin', '+master:new-branch')\n    origin.checkout('new-branch')\n    ok_(op.exists(op.join(orig_path, filename)))\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_remote_update(path1, path2, path3):\n\n    git1 = GitRepo(path1)\n    git2 = GitRepo(path2)\n    git3 = GitRepo(path3)\n\n    git1.add_remote('git2', path2)\n    git1.add_remote('git3', path3)\n\n    # Setting up remote 'git2'\n    with open(op.join(path2, 'masterfile'), 'w') as f:\n        f.write(\"git2 in master\")\n    git2.add('masterfile')\n    git2.commit(\"Add something to master.\")\n    git2.checkout('branch2', ['-b'])\n    with open(op.join(path2, 'branch2file'), 'w') as f:\n        f.write(\"git2 in branch2\")\n    git2.add('branch2file')\n    git2.commit(\"Add something to branch2.\")\n\n    # Setting up remote 'git3'\n    with open(op.join(path3, 'masterfile'), 'w') as f:\n        f.write(\"git3 in master\")\n    git3.add('masterfile')\n    git3.commit(\"Add something to master.\")\n    git3.checkout('branch3', ['-b'])\n    with open(op.join(path3, 'branch3file'), 'w') as f:\n        f.write(\"git3 in branch3\")\n    git3.add('branch3file')\n    git3.commit(\"Add something to branch3.\")\n\n    git1.update_remote()\n\n    # checkouts are 'tests' themselves, since they'll raise CommandError\n    # if something went wrong\n    git1.checkout('branch2')\n    git1.checkout('branch3')\n\n    branches1 = git1.get_branches()\n    eq_({'branch2', 'branch3'}, set(branches1))\n\n\n# TODO: Why was it \"flavors=local_testrepo_flavors\" ? What's the windows issue here?\n@with_testrepos('.*git.*', flavors=['clone'])\n@with_tempfile\ndef test_GitRepo_get_files(url, path):\n\n    gr = GitRepo.clone(url, path)\n\n    # get the expected files via os for comparison:\n    os_files = set()\n    for (dirpath, dirnames, filenames) in os.walk(path):\n        rel_dir = os.path.relpath(dirpath, start=path)\n        if rel_dir.startswith(\".git\"):\n            continue\n        for file_ in filenames:\n            file_path = os.path.normpath(op.join(rel_dir, file_))\n            os_files.add(file_path)\n\n    # get the files via GitRepo:\n    local_files = set(gr.get_files())\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n\n    eq_(local_files, set(gr.get_indexed_files()))\n    eq_(local_files, remote_files)\n    eq_(local_files, os_files)\n\n    # create a different branch:\n    gr.checkout('new_branch', ['-b'])\n    filename = 'another_file.dat'\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"something\")\n    gr.add(filename)\n    gr.commit(\"Added.\")\n\n    # now get the files again:\n    local_files = set(gr.get_files())\n    eq_(local_files, os_files.union({filename}))\n    # retrieve remote branch again, which should not have changed:\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n    eq_(remote_files, os_files)\n    eq_(set([filename]), local_files.difference(remote_files))\n\n    # switch back and query non-active branch:\n    gr.checkout('master')\n    local_files = set(gr.get_files())\n    branch_files = set(gr.get_files(branch=\"new_branch\"))\n    eq_(set([filename]), branch_files.difference(local_files))\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:505\n@known_failure_windows\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_GitRepo_get_toppath(repo, tempdir, repo2):\n    reporeal = op.realpath(repo)\n    eq_(GitRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(GitRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    GitRepo(repo2, create=True)\n    repo2real = op.realpath(repo2)\n    nested = op.join(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(GitRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(GitRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(GitRepo.get_toppath(tempdir), None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_dirty(path):\n\n    repo = GitRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # untracked file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt')\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(not repo.dirty)\n    # modified file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt')\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # An empty directory doesn't count as dirty.\n    os.mkdir(op.join(path, \"empty\"))\n    ok_(not repo.dirty)\n    # Neither does an empty directory with an otherwise empty directory.\n    os.mkdir(op.join(path, \"empty\", \"empty-again\"))\n    ok_(not repo.dirty)\n\n    subm = GitRepo(repo.pathobj / \"subm\", create=True)\n    (subm.pathobj / \"subfile\").write_text(u\"\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n    (subm.pathobj / \"subfile\").write_text(u\"changed\")\n    ok_(repo.dirty)\n\n    # User configuration doesn't affect .dirty's answer.\n    repo.config.set(\"diff.ignoreSubmodules\", \"all\", where=\"local\")\n    ok_(repo.dirty)\n    # GitRepo.commit currently can't handle this setting, so remove it for the\n    # save() calls below.\n    repo.config.unset(\"diff.ignoreSubmodules\", where=\"local\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n\n    repo.config.set(\"status.showUntrackedFiles\", \"no\", where=\"local\")\n    create_tree(repo.path, {\"untracked_dir\": {\"a\": \"a\"}})\n    ok_(repo.dirty)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_merge_base(src):\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    assert_raises(ValueError, repo.get_merge_base, [])\n    branch1 = repo.get_active_branch()\n    branch1_hexsha = repo.get_hexsha()\n    eq_(len(branch1_hexsha), 40)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # Let's create a detached branch\n    branch2 = \"_detach_\"\n    repo.checkout(branch2, options=[\"--orphan\"])\n    # it will have all the files\n    # Must not do:  https://github.com/gitpython-developers/GitPython/issues/375\n    # repo.git_add('.')\n    repo.add('*')\n    # NOTE: fun part is that we should have at least a different commit message\n    # so it results in a different checksum ;)\n    repo.commit(\"committing again\")\n    assert(repo.get_indexed_files())  # we did commit\n    assert(repo.get_merge_base(branch1) is None)\n    assert(repo.get_merge_base([branch2, branch1]) is None)\n\n    # Let's merge them up -- then merge base should match the master\n    repo.merge(branch1, allow_unrelated=True)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # if points to some empty/non-existing branch - should also be None\n    assert(repo.get_merge_base(['nonexistent', branch2]) is None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_git_get_branch_commits_(src):\n\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    commits_default = list(repo.get_branch_commits_())\n    commits = list(repo.get_branch_commits_('master'))\n    eq_(commits, commits_default)\n    eq_(len(commits), 1)\n\n\n@with_testrepos(flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_get_tracking_branch(o_path, c_path):\n\n    clone = GitRepo.clone(o_path, c_path)\n    # Note, that the default branch might differ even if it is always 'master'.\n    # For direct mode annex repositories it would then be \"annex/direct/master\"\n    # for example. Therefore use whatever branch is checked out by default:\n    master_branch = clone.get_active_branch()\n    ok_(master_branch)\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n\n    clone.checkout('new_branch', ['-b'])\n\n    eq_((None, None), clone.get_tracking_branch())\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch(master_branch))\n\n    clone.checkout(master_branch, options=[\"--track\", \"-btopic\"])\n    eq_(('.', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n    eq_((None, None),\n        clone.get_tracking_branch(remote_only=True))\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_submodule_deinit(path):\n    from datalad.support.annexrepo import AnnexRepo\n\n    top_repo = AnnexRepo(path, create=False)\n    eq_({'subm 1', '2'}, {s.name for s in top_repo.get_submodules()})\n    # note: here init=True is ok, since we are using it just for testing\n    with swallow_logs(new_level=logging.WARN) as cml:\n        top_repo.update_submodule('subm 1', init=True)\n        assert_in('Do not use update_submodule with init=True', cml.out)\n    top_repo.update_submodule('2', init=True)\n\n    # ok_(all([s.module_exists() for s in top_repo.get_submodules()]))\n    # TODO: old assertion above if non-bare? (can't use \"direct mode\" in test_gitrepo)\n    # Alternatively: New testrepo (plain git submodules) and have a dedicated\n    # test for annexes in addition\n    ok_(all([GitRepo.is_valid_repo(op.join(top_repo.path, s.path))\n             for s in top_repo.get_submodules()]))\n\n    # modify submodule:\n    with open(op.join(top_repo.path, 'subm 1', 'file_ut.dat'), \"w\") as f:\n        f.write(\"some content\")\n\n    assert_raises(CommandError, top_repo.deinit_submodule, 'sub1')\n\n    # using force should work:\n    top_repo.deinit_submodule('subm 1', force=True)\n\n    ok_(not top_repo.repo.submodule('subm 1').module_exists())\n\n\n@with_testrepos(\".*basic_git.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_GitRepo_add_submodule(source, path):\n\n    top_repo = GitRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n    ok_clean_git(path)\n    ok_clean_git(op.join(path, 'sub'))\n\n\ndef test_GitRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef check_update_submodule_init_adjust_branch(is_ancestor, path):\n    src = GitRepo(op.join(path, \"src\"), create=True)\n    src_sub = GitRepo(op.join(src.path, \"sub\"), create=True)\n    src_sub.commit(msg=\"c0\", options=[\"--allow-empty\"])\n    src_sub.commit(msg=\"c1\", options=[\"--allow-empty\"])\n    src.add_submodule('sub', name='sub')\n    src.commit(msg=\"Add submodule\")\n\n    # Move subdataset past the registered commit...\n    hexsha_registered = src_sub.get_hexsha()\n    if is_ancestor:\n        # ... where the registered commit is an ancestor of the new one.\n        src_sub.commit(msg=\"c2\", options=[\"--allow-empty\"])\n    else:\n        # ... where the registered commit is NOT an ancestor of the new one.\n        src_sub.call_git([\"reset\", \"--hard\", \"master~1\"])  # c0\n    hexsha_sub = src_sub.get_hexsha()\n\n    clone = GitRepo.clone(url=src.path,\n                          path=op.join(path, \"clone\"),\n                          create=True)\n    clone_sub = GitRepo.clone(url=src_sub.path,\n                              path=op.join(clone.path, \"sub\"),\n                              create=True)\n    ok_(clone.dirty)\n    eq_(clone_sub.get_active_branch(), \"master\")\n    eq_(hexsha_sub, clone_sub.get_hexsha())\n\n    clone.update_submodule(\"sub\", init=True)\n\n    assert_false(clone.dirty)\n    eq_(hexsha_registered, clone_sub.get_hexsha())\n    if is_ancestor:\n        eq_(clone_sub.get_active_branch(), \"master\")\n    else:\n        assert_false(clone_sub.get_active_branch())\n\n\ndef test_GitRepo_update_submodule_init_adjust_branch():\n    yield check_update_submodule_init_adjust_branch, True\n    yield check_update_submodule_init_adjust_branch, False\n\n\ndef test_GitRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile\ndef test_get_submodules_parent_on_unborn_branch(path):\n    repo = GitRepo(path, create=True)\n    subrepo = GitRepo(op.join(path, \"sub\"), create=True)\n    subrepo.commit(msg=\"s\", options=[\"--allow-empty\"])\n    repo.add_submodule(path=\"sub\")\n    eq_([s.name for s in repo.get_submodules()],\n        [\"sub\"])\n\n\ndef test_to_options():\n\n    class Some(object):\n\n        def cmd_func(self, git_options=None, annex_options=None, options=None):\n\n            git_options = git_options[:] if git_options else []\n            annex_options = annex_options[:] if annex_options else []\n            options = options[:] if options else []\n\n            faked_cmd_call = ['git'] + git_options + ['annex'] + \\\n                             annex_options + ['my_cmd'] + options\n\n            return faked_cmd_call\n\n    eq_(Some().cmd_func(options=to_options(m=\"bla\", force=True)),\n        ['git', 'annex', 'my_cmd', '--force', '-m', 'bla'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\"),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C', '/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\", split_single_char_options=False),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n\ndef test_to_options_from_gitpython():\n    \"\"\"Imported from GitPython and modified.\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    eq_([\"-s\"], to_options(**{'s': True}))\n    eq_([\"-s\", \"5\"], to_options(**{'s': 5}))\n    eq_([], to_options(**{'s': None}))\n\n    eq_([\"--max-count\"], to_options(**{'max_count': True}))\n    eq_([\"--max-count=5\"], to_options(**{'max_count': 5}))\n    eq_([\"--max-count=0\"], to_options(**{'max_count': 0}))\n    eq_([], to_options(**{'max_count': None}))\n\n    # Multiple args are supported by using lists/tuples\n    eq_([\"-L\", \"1-3\", \"-L\", \"12-18\"], to_options(**{'L': ('1-3', '12-18')}))\n    eq_([\"-C\", \"-C\"], to_options(**{'C': [True, True, None, False]}))\n\n    # order is undefined\n    res = to_options(**{'s': True, 't': True})\n    eq_({'-s', '-t'}, set(res))\n\n\n@with_tempfile\ndef test_GitRepo_count_objects(repo_path):\n\n    repo = GitRepo(repo_path, create=True)\n    # test if dictionary returned\n    eq_(isinstance(repo.count_objects, dict), True)\n    # test if dictionary contains keys and values we expect\n    empty_count = {'count': 0, 'garbage': 0,  'in-pack': 0, 'packs': 0, 'prune-packable': 0,\n                   'size': 0, 'size-garbage': 0, 'size-pack': 0}\n    eq_(empty_count, repo.count_objects)\n\n\n# this is simply broken on win, but less important\n# https://github.com/datalad/datalad/issues/3639\n@skip_if_on_windows\n@with_tempfile\ndef test_optimized_cloning(path):\n    # make test repo with one file and one commit\n    originpath = op.join(path, 'origin')\n    repo = GitRepo(originpath, create=True)\n    with open(op.join(originpath, 'test'), 'w') as f:\n        f.write('some')\n    repo.add('test')\n    repo.commit('init')\n    ok_clean_git(originpath, annex=False)\n    from glob import glob\n\n    def _get_inodes(repo):\n        return dict(\n            [(os.path.join(*o.split(os.sep)[-2:]),\n              os.stat(o).st_ino)\n             for o in glob(os.path.join(repo.path,\n                                        repo.get_git_dir(repo),\n                                        'objects', '*', '*'))])\n\n    origin_inodes = _get_inodes(repo)\n    # now clone it in different ways and see what happens to the object storage\n    from datalad.support.network import get_local_file_url\n    clonepath = op.join(path, 'clone')\n    for src in (originpath, get_local_file_url(originpath, compatibility='git')):\n        # deprecated\n        assert_raises(DeprecatedError, GitRepo, url=src, path=clonepath)\n        clone = GitRepo.clone(url=src, path=clonepath, create=True)\n        clone_inodes = _get_inodes(clone)\n        eq_(origin_inodes, clone_inodes, msg='with src={}'.format(src))\n        rmtree(clonepath)\n#        del clone\n#        gc.collect()\n        # Note: del needed, since otherwise WeakSingletonRepo would just\n        # return the original object in second run\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_gitpy_injection(path, path2):\n\n    gr = GitRepo(path, create=True)\n    gr._GIT_COMMON_OPTIONS.extend(['test-option'])\n\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # once set, these option should be persistent across git calls:\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.another_unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # but other repos should not be affected:\n    gr2 = GitRepo(path2, create=True)\n    with assert_raises(GitCommandError) as cme:\n        gr2.repo.git.unknown_git_command()\n    assert_not_in('test-option', exc_str(cme.exception))\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_GitRepo_flyweight(path1, path2):\n\n    repo1 = GitRepo(path1, create=True)\n    assert_is_instance(repo1, GitRepo)\n    # instantiate again:\n    repo2 = GitRepo(path1, create=False)\n    assert_is_instance(repo2, GitRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in a different way:\n    with chpwd(path1):\n        repo3 = GitRepo(op.relpath(path1, start=path2), create=False)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # and realpath attribute is the same, so they are still equal:\n    ok_(repo1 == repo3)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_GitRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(GitRepo, path, store)\n\n\n@with_tree(tree={'ignore-sub.me': {'a_file.txt': 'some content'},\n                 'ignore.me': 'ignored content',\n                 'dontigno.re': 'other content'})\ndef test_GitRepo_gitignore(path):\n\n    gr = GitRepo(path, create=True)\n    sub = GitRepo(op.join(path, 'ignore-sub.me'))\n    # we need to commit something, otherwise add_submodule\n    # will already refuse the submodule for having no commit\n    sub.add('a_file.txt')\n    sub.commit()\n\n    from ..exceptions import GitIgnoreError\n\n    with open(op.join(path, '.gitignore'), \"w\") as f:\n        f.write(\"*.me\")\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add('ignore.me')\n    eq_(cme.exception.paths, ['ignore.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add_submodule(path='ignore-sub.me')\n    eq_(cme.exception.paths, ['ignore-sub.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add(['ignore.me', 'dontigno.re', op.join('ignore-sub.me', 'a_file.txt')])\n    eq_(set(cme.exception.paths), {'ignore.me', 'ignore-sub.me'})\n\n    eq_(gr.get_gitattributes('.')['.'], {})  # nothing is recorded within .gitattributes\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_set_remote_url(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add_remote('some', 'http://example.com/.git')\n    eq_(gr.config['remote.some.url'],\n                 'http://example.com/.git')\n    # change url:\n    gr.set_remote_url('some', 'http://believe.it')\n    eq_(gr.config['remote.some.url'],\n                 'http://believe.it')\n\n    # set push url:\n    gr.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    eq_(gr.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n\n    # add remote without url\n    url2 = 'http://repo2.example.com/.git'\n    gr.add_remote('some-without-url', url2)\n    eq_(gr.config['remote.some-without-url.url'], url2)\n    # \"remove\" it\n    gr.config.unset('remote.some-without-url.url', where='local')\n    with assert_raises(KeyError):\n        gr.config['remote.some-without-url.url']\n    eq_(set(gr.get_remotes()), {'some', 'some-without-url'})\n    eq_(set(gr.get_remotes(with_urls_only=True)), {'some'})\n\n\n@with_tempfile(mkdir=True)\ndef test_gitattributes(path):\n    gr = GitRepo(path, create=True)\n    # starts without any attributes file\n    ok_(not op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {})\n    # bool is a tag or unsets, anything else is key/value\n    gr.set_gitattributes([('*', {'tag': True}), ('*', {'sec.key': 'val'})])\n    ok_(op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {'tag': True, 'sec.key': 'val'})\n    # unset by amending the record, but does not remove notion of the\n    # tag entirely\n    gr.set_gitattributes([('*', {'tag': False})])\n    eq_(gr.get_gitattributes('.')['.'], {'tag': False, 'sec.key': 'val'})\n    # attributes file is not added or commited, we can ignore such\n    # attributes\n    eq_(gr.get_gitattributes('.', index_only=True)['.'], {})\n\n    # we can send absolute path patterns and write to any file, and\n    # the patterns will be translated relative to the target file\n    gr.set_gitattributes([\n        (op.join(gr.path, 'relative', 'ikethemike/**'), {'bang': True})],\n        attrfile=op.join('relative', '.gitattributes'))\n    # directory and file get created\n    ok_(op.exists(op.join(gr.path, 'relative', '.gitattributes')))\n    eq_(gr.get_gitattributes(\n        op.join(gr.path, 'relative', 'ikethemike', 'probe')),\n        # always comes out relative to the repo root, even if abs goes in\n        {op.join('relative', 'ikethemike', 'probe'):\n            {'tag': False, 'sec.key': 'val', 'bang': True}})\n    if get_encoding_info()['default'] != 'ascii' and not on_windows:\n        # do not perform this on obscure systems without anything like UTF\n        # it is not relevant whether a path actually exists, and paths\n        # with spaces and other funky stuff are just fine\n        funky = u'{} {}'.format(\n            get_most_obscure_supported_name(),\n            get_most_obscure_supported_name())\n        gr.set_gitattributes([(funky, {'this': 'that'})])\n        eq_(gr.get_gitattributes(funky)[funky], {\n            'this': 'that',\n            'tag': False,\n            'sec.key': 'val',\n        })\n\n    # mode='w' should replace the entire file:\n    gr.set_gitattributes([('**', {'some': 'nonsense'})], mode='w')\n    eq_(gr.get_gitattributes('.')['.'], {'some': 'nonsense'})\n\n\n@with_tempfile(mkdir=True)\ndef test_get_hexsha_tag(path):\n    gr = GitRepo(path, create=True)\n    gr.commit(msg=\"msg\", options=[\"--allow-empty\"])\n    gr.tag(\"atag\", message=\"atag msg\")\n    # get_hexsha() dereferences a tag to a commit.\n    eq_(gr.get_hexsha(\"atag\"), gr.get_hexsha())\n\n\n@with_tempfile(mkdir=True)\ndef test_get_tags(path):\n    from unittest.mock import patch\n\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_tags(), [])\n    eq_(gr.describe(), None)\n\n    # Explicitly override the committer date because tests may set it to a\n    # fixed value, but we want to check that the returned tags are sorted by\n    # the date the tag (for annotaged tags) or commit (for lightweight tags)\n    # was created.\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Thu, 07 Apr 2005 22:13:13 +0200\"}):\n        create_tree(gr.path, {'file': \"\"})\n        gr.add('file')\n        gr.commit(msg=\"msg\")\n        eq_(gr.get_tags(), [])\n        eq_(gr.describe(), None)\n\n        gr.tag(\"nonannotated\")\n        tags1 = [{'name': 'nonannotated', 'hexsha': gr.get_hexsha()}]\n        eq_(gr.get_tags(), tags1)\n        eq_(gr.describe(), None)\n        eq_(gr.describe(tags=True), tags1[0]['name'])\n\n    first_commit = gr.get_hexsha()\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 08 Apr 2005 22:13:13 +0200\"}):\n\n        create_tree(gr.path, {'file': \"123\"})\n        gr.add('file')\n        gr.commit(msg=\"changed\")\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 09 Apr 2005 22:13:13 +0200\"}):\n        gr.tag(\"annotated\", message=\"annotation\")\n    # The annotated tag happened later, so it comes last.\n    tags2 = tags1 + [{'name': 'annotated', 'hexsha': gr.get_hexsha()}]\n    eq_(gr.get_tags(), tags2)\n    eq_(gr.describe(), tags2[1]['name'])\n\n    # compare prev commit\n    eq_(gr.describe(commitish=first_commit), None)\n    eq_(gr.describe(commitish=first_commit, tags=True), tags1[0]['name'])\n\n    gr.tag('specific', commit='HEAD~1')\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD~1'))\n    assert_in('specific', gr.get_tags(output='name'))\n\n    # retag a different commit\n    assert_raises(CommandError, gr.tag, 'specific', commit='HEAD')\n    # force it\n    gr.tag('specific', commit='HEAD', options=['-f'])\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD'))\n\n    # delete\n    gr.call_git(['tag', '-d', 'specific'])\n    eq_(gr.get_tags(), tags2)\n    # more than one\n    gr.tag('one')\n    gr.tag('two')\n    gr.call_git(['tag', '-d', 'one', 'two'])\n    eq_(gr.get_tags(), tags2)\n\n\n@with_tree(tree={'1': \"\"})\ndef test_get_commit_date(path):\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_commit_date(), None)\n\n    # Let's make a commit with a custom date\n    DATE = \"Wed Mar 14 03:47:30 2018 -0000\"\n    DATE_EPOCH = 1520999250\n    gr.add('1')\n    gr.commit(\"committed\", date=DATE)\n    gr = GitRepo(path, create=True)\n    date = gr.get_commit_date()\n    neq_(date, None)\n    eq_(date, DATE_EPOCH)\n\n    eq_(date, gr.get_commit_date('master'))\n    # and even if we get into a detached head\n    gr.checkout(gr.get_hexsha())\n    eq_(gr.get_active_branch(), None)\n    eq_(date, gr.get_commit_date('master'))\n\n\n@with_tree(tree={\"foo\": \"foo content\",\n                 \"bar\": \"bar content\"})\ndef test_fake_dates(path):\n    gr = GitRepo(path, create=True, fake_dates=True)\n\n    gr.add(\"foo\")\n    gr.commit(\"commit foo\")\n\n    seconds_initial = gr.config.obtain(\"datalad.fake-dates-start\")\n\n    # First commit is incremented by 1 second.\n    eq_(seconds_initial + 1, gr.get_commit_date())\n\n    # The second commit by 2.\n    gr.add(\"bar\")\n    gr.commit(\"commit bar\")\n    eq_(seconds_initial + 2, gr.get_commit_date())\n\n    # If we checkout another branch, its time is still based on the latest\n    # timestamp in any local branch.\n    gr.checkout(\"other\", options=[\"--orphan\"])\n    with open(op.join(path, \"baz\"), \"w\") as ofh:\n        ofh.write(\"baz content\")\n    gr.add(\"baz\")\n    gr.commit(\"commit baz\")\n    eq_(gr.get_active_branch(), \"other\")\n    eq_(seconds_initial + 3, gr.get_commit_date())\n\n\ndef test_guard_BadName():\n    from gitdb.exc import BadName\n\n    calls = []\n\n    class Vulnerable(object):\n        def precommit(self):\n            calls.append('precommit')\n\n        @guard_BadName\n        def __call__(self, x, y=2):\n            if not calls:\n                calls.append(1)\n                raise BadName\n            return x+y\n    v = Vulnerable()\n    eq_(v(1, y=3), 4)\n    eq_(calls, [1, 'precommit'])\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\ndef test_custom_runner_protocol(path):\n    # Check that a runner with a non-default protocol gets wired up correctly.\n    prot = ExecutionTimeProtocol()\n    gr = GitRepo(path, runner=Runner(cwd=path, protocol=prot), create=True)\n\n    ok_(len(prot) > 0)\n    ok_(prot[0]['duration'] >= 0)\n\n    def check(prev_len, prot, command):\n        # Check that the list grew and has the expected command without\n        # assuming that it gained _only_ a one command.\n        ok_(len(prot) > prev_len)\n        assert_in(command,\n                  sum([p[\"command\"] for p in prot[prev_len:]], []))\n\n    prev_len = len(prot)\n    gr.add(\"foo\")\n    check(prev_len, prot, \"add\")\n\n    # commit no longer uses a Runner with protocol capabilities\n    #prev_len = len(prot)\n    #gr.commit(\"commit foo\")\n    #check(prev_len, prot, \"commit\")\n\n    ok_(all(p['duration'] >= 0 for p in prot))\n\n\n@with_tempfile(mkdir=True)\ndef test_duecredit(path):\n    # Just to check that no obvious side-effects\n    run = Runner(cwd=path).run\n    cmd = [\n        sys.executable, \"-c\",\n        \"from datalad.support.gitrepo import GitRepo; GitRepo(%r, create=True)\" % path\n    ]\n\n    env = os.environ.copy()\n\n    # Test with duecredit not enabled for sure\n    env.pop('DUECREDIT_ENABLE', None)\n    # Alternative workaround for what to be fixed by\n    # https://github.com/datalad/datalad/pull/3215\n    # where underlying datalad process might issue a warning since our temp\n    # cwd is not matching possibly present PWD env variable\n    env.pop('PWD', None)\n\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err  # Let's not depend on where duecredit decides to spit out\n    # All quiet\n    eq_(outs, '')\n\n    # and now enable DUECREDIT - output could come to stderr\n    env['DUECREDIT_ENABLE'] = '1'\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err\n\n    if external_versions['duecredit']:\n        assert_in('Data management and distribution platform', outs)\n    else:\n        eq_(outs, '')\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_revisions(path):\n    gr = GitRepo(path, create=True)\n\n    def commit(msg):\n        gr.commit(msg=msg, options=[\"--allow-empty\"])\n\n    # We catch the error and return empty if the current branch doesn't have a\n    # commit checked out.\n    eq_(gr.get_revisions(), [])\n\n    # But will raise if on a bad ref name, including an unborn branch.\n    with assert_raises(CommandError):\n        gr.get_revisions(\"master\")\n\n    # By default, we query HEAD.\n    commit(\"1\")\n    eq_(len(gr.get_revisions()), 1)\n\n    gr.checkout(\"other\", options=[\"-b\"])\n    commit(\"2\")\n\n    # We can also query branch by name.\n    eq_(len(gr.get_revisions(\"master\")), 1)\n    eq_(len(gr.get_revisions(\"other\")), 2)\n\n    # \"name\" is sugar for [\"name\"].\n    eq_(gr.get_revisions(\"master\"),\n        gr.get_revisions([\"master\"]))\n\n    gr.checkout(\"master\")\n    commit(\"3\")\n    eq_(len(gr.get_revisions(\"master\")), 2)\n    # We can pass multiple revisions...\n    eq_(len(gr.get_revisions([\"master\", \"other\"])), 3)\n    # ... or options like --all and --branches\n    eq_(gr.get_revisions([\"master\", \"other\"]),\n        gr.get_revisions(options=[\"--all\"]))\n\n    # Ranges are supported.\n    eq_(gr.get_revisions(\"master..\"), [])\n\n\n@with_tree({\"foo\": \"foo\"})\ndef test_gitrepo_add_to_git_with_annex_v7(path):\n    from datalad.support.annexrepo import AnnexRepo\n    ar = AnnexRepo(path, create=True, version=7)\n    gr = GitRepo(path)\n    gr.add(\"foo\")\n    gr.commit(msg=\"c1\")\n    assert_false(ar.is_under_annex(\"foo\"))\n\n\n@with_tree({\"foo\": \"foo\", \"bar\": \"bar\"})\ndef test_gitrepo_call_git_methods(path):\n    gr = GitRepo(path)\n    gr.add([\"foo\", \"bar\"])\n    gr.commit(msg=\"foobar\")\n    gr.call_git([\"mv\"], files=[\"foo\", \"foo.txt\"])\n    ok_(op.exists(op.join(gr.path, 'foo.txt')))\n\n    for expect_fail, check in [(False, assert_in),\n                               (True, assert_not_in)]:\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            with assert_raises(CommandError):\n                gr.call_git([\"mv\"], files=[\"notthere\", \"dest\"],\n                            expect_fail=expect_fail)\n            check(\"notthere\", cml.out)\n\n    eq_(list(gr.call_git_items_([\"ls-files\"])),\n        [\"bar\", \"foo.txt\"])\n    eq_(list(gr.call_git_items_([\"ls-files\", \"-z\"], sep=\"\\0\")),\n        # Note: The custom separator has trailing empty item, but this is an\n        # arbitrary command with unknown output it isn't safe to trim it.\n        [\"bar\", \"foo.txt\", \"\"])\n\n    with assert_raises(AssertionError):\n        gr.call_git_oneline([\"ls-files\"])\n\n    eq_(gr.call_git_oneline([\"ls-files\"], files=[\"bar\"]),\n        \"bar\")\n\n    ok_(gr.call_git_success([\"rev-parse\", \"HEAD^{commit}\"]))\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        assert_false(gr.call_git_success([\"rev-parse\", \"HEAD^{blob}\"]))\n        assert_not_in(\"blob\", cml.out)\n\n\n@skip_if_no_network\n@with_tempfile\ndef _test_protocols(proto, destdir):\n    GitRepo.clone('%s://github.com/datalad-tester/testtt' % proto, destdir)\n\n\n@integration\ndef test_protocols():\n    # git-annex-standalone build can get git bundle which would fail to\n    # download via https, resulting in messages such as\n    #  fatal: unable to find remote helper for 'https'\n    # which happened with git-annex-standalone 7.20191017+git2-g7b13db551-1~ndall+1\n\n    # http is well tested already\n    for proto in 'git', 'https':\n        yield _test_protocols, proto\n",
            "header": {
                "file": {
                    "path": "datalad/support/tests/test_gitrepo.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "e4aa6638b",
                "new_commit_hash": "b13af7f25",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/tests/test_gitrepo.py"
            },
            "plus_file": {
                "path": "b/datalad/support/tests/test_gitrepo.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 1080,
                            "length": 6
                        },
                        "new_range": {
                            "start": 1080,
                            "length": 32
                        },
                        "section": "def test_to_options():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        ['git', '-C/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def test_to_options_from_gitpython():",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"Imported from GitPython and modified.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    Original copyright:",
                                "type": "added"
                            },
                            {
                                "content": "        Copyright (C) 2008, 2009 Michael Trier and contributors",
                                "type": "added"
                            },
                            {
                                "content": "    Original license:",
                                "type": "added"
                            },
                            {
                                "content": "        BSD 3-Clause \"New\" or \"Revised\" License",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"-s\"], to_options(**{'s': True}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"-s\", \"5\"], to_options(**{'s': 5}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([], to_options(**{'s': None}))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"--max-count\"], to_options(**{'max_count': True}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"--max-count=5\"], to_options(**{'max_count': 5}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"--max-count=0\"], to_options(**{'max_count': 0}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([], to_options(**{'max_count': None}))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # Multiple args are supported by using lists/tuples",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"-L\", \"1-3\", \"-L\", \"12-18\"], to_options(**{'L': ('1-3', '12-18')}))",
                                "type": "added"
                            },
                            {
                                "content": "    eq_([\"-C\", \"-C\"], to_options(**{'C': [True, True, None, False]}))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    # order is undefined",
                                "type": "added"
                            },
                            {
                                "content": "    res = to_options(**{'s': True, 't': True})",
                                "type": "added"
                            },
                            {
                                "content": "    eq_({'-s', '-t'}, set(res))",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "@with_tempfile",
                                "type": "context"
                            },
                            {
                                "content": "def test_GitRepo_count_objects(repo_path):",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/support/tests/test_gitrepo.py",
                            "type": "function",
                            "name": "test_to_options_from_gitpython",
                            "content": "def test_to_options_from_gitpython():\n    \"\"\"Imported from GitPython and modified.\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    eq_([\"-s\"], to_options(**{'s': True}))\n    eq_([\"-s\", \"5\"], to_options(**{'s': 5}))\n    eq_([], to_options(**{'s': None}))\n\n    eq_([\"--max-count\"], to_options(**{'max_count': True}))\n    eq_([\"--max-count=5\"], to_options(**{'max_count': 5}))\n    eq_([\"--max-count=0\"], to_options(**{'max_count': 0}))\n    eq_([], to_options(**{'max_count': None}))\n\n    # Multiple args are supported by using lists/tuples\n    eq_([\"-L\", \"1-3\", \"-L\", \"12-18\"], to_options(**{'L': ('1-3', '12-18')}))\n    eq_([\"-C\", \"-C\"], to_options(**{'C': [True, True, None, False]}))\n\n    # order is undefined\n    res = to_options(**{'s': True, 't': True})\n    eq_({'-s', '-t'}, set(res))",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 1083,
                            "end_lineno": 1106,
                            "parent": null
                        }
                    ],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "8fb58b5da3dcb5ca940664b159653baa9be85133^",
    "new_commit_hash": "8fb58b5da3dcb5ca940664b159653baa9be85133",
    "commit_message": "Merge pull request #4176 from mih/rf-tooptions\n\nRF: Import GitPython code used for to_options()",
    "commit_date": "2020-02-21T05:21:30+01:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "8fb58b5da3dcb5ca940664b159653baa9be85133",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class GitRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import assert_is_instance\n\nimport logging\n\nimport os\nimport os.path as op\n\nimport sys\n\n\nfrom datalad import get_encoding_info\nfrom datalad.cmd import Runner\n\nfrom datalad.utils import (\n    chpwd,\n    getpwd,\n    on_windows,\n    rmtree,\n    unlink,\n)\nfrom datalad.tests.utils import (\n    assert_cwd_unchanged,\n    assert_false,\n    assert_in,\n    assert_in_results,\n    assert_not_in,\n    assert_raises,\n    assert_re_in,\n    create_tree,\n    eq_,\n    get_most_obscure_supported_name,\n    integration,\n    known_failure_windows,\n    local_testrepo_flavors,\n    neq_,\n    ok_,\n    ok_clean_git,\n    skip_if,\n    skip_if_no_network,\n    skip_if_on_windows,\n    skip_ssh,\n    SkipTest,\n    swallow_logs,\n    with_tempfile,\n    with_testrepos,\n    with_tree,\n)\nfrom datalad.tests.utils_testrepos import BasicAnnexTestRepo\n\nfrom datalad.dochelpers import exc_str\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.support.gitrepo import (\n    _normalize_path,\n    GitCommandError,\n    gitpy,\n    GitRepo,\n    guard_BadName,\n    InvalidGitRepositoryError,\n    normalize_paths,\n    NoSuchPathError,\n    to_options,\n)\nfrom datalad.support.exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    PathKnownToRepositoryError,\n)\nfrom datalad.support.external_versions import external_versions\nfrom datalad.support.protocol import ExecutionTimeProtocol\nfrom datalad.support.tests.utils import check_repo_deals_with_inode_change\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_invalid_path(path):\n    with chpwd(path):\n        assert_raises(ValueError, GitRepo, path=\"git://some/url\", create=True)\n        ok_(not op.exists(op.join(path, \"git:\")))\n        assert_raises(ValueError, GitRepo, path=\"file://some/relative/path\", create=True)\n        ok_(not op.exists(op.join(path, \"file:\")))\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_instance_from_clone(src, dst):\n\n    gr = GitRepo.clone(src, dst)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    assert_is_instance(gr.repo, gitpy.Repo,\n                       \"Failed to instantiate GitPython Repo object.\")\n    ok_(op.exists(op.join(dst, '.git')))\n\n    # do it again should raise GitCommandError since git will notice there's\n    # already a git-repo at that path and therefore can't clone to `dst`\n    # Note: Since GitRepo is now a WeakSingletonRepo, this is prevented from\n    # happening atm. Disabling for now:\n#    raise SkipTest(\"Disabled for RF: WeakSingletonRepo\")\n    with swallow_logs() as logs:\n        assert_raises(GitCommandError, GitRepo.clone, src, dst)\n\n\n@assert_cwd_unchanged\n@with_testrepos(flavors=local_testrepo_flavors)\ndef test_GitRepo_instance_from_existing(path):\n\n    gr = GitRepo(path)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_instance_from_not_existing(path, path2):\n    # 1. create=False and path doesn't exist:\n    assert_raises(NoSuchPathError, GitRepo, path, create=False)\n    assert_false(op.exists(path))\n\n    # 2. create=False, path exists, but no git repo:\n    os.mkdir(path)\n    ok_(op.exists(path))\n    assert_raises(InvalidGitRepositoryError, GitRepo, path, create=False)\n    assert_false(op.exists(op.join(path, '.git')))\n\n    # 3. create=True, path doesn't exist:\n    gr = GitRepo(path2, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path2, '.git')))\n    ok_clean_git(path2, annex=False)\n\n    # 4. create=True, path exists, but no git repo:\n    gr = GitRepo(path, create=True)\n    assert_is_instance(gr, GitRepo, \"GitRepo was not created.\")\n    ok_(op.exists(op.join(path, '.git')))\n    ok_clean_git(path, annex=False)\n\n\n@with_tempfile\ndef test_GitRepo_init_options(path):\n    # passing an option, not explicitly defined in GitRepo class:\n    gr = GitRepo(path, create=True, bare=True)\n\n    cfg = gr.repo.config_reader()\n    ok_(cfg.get_value(section=\"core\", option=\"bare\"))\n\n\n@skip_if(external_versions['cmd:git'] < '2.14.0')\n@with_tree(\n    tree={\n        'subds': {\n            'file_name': ''\n        }\n    }\n)\ndef test_init_fail_under_known_subdir(path):\n    repo = GitRepo(path, create=True)\n    repo.add(op.join('subds', 'file_name'))\n    # Should fail even if we do not commit but only add to index:\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n    assert_in(\"file_name\", str(cme.exception))  # we provide a list of offenders\n    # and after we commit - the same story\n    repo.commit(\"added file\")\n    with assert_raises(PathKnownToRepositoryError) as cme:\n        GitRepo(op.join(path, 'subds'), create=True)\n\n    # But it would succeed if we disable the checks\n    GitRepo(op.join(path, 'subds'), create=True, create_sanity_checks=False)\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_equals(path1, path2):\n\n    repo1 = GitRepo(path1)\n    repo2 = GitRepo(path1)\n    ok_(repo1 == repo2)\n    eq_(repo1, repo2)\n    repo2 = GitRepo(path2)\n    neq_(repo1, repo2)\n    ok_(repo1 != repo2)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:515\n@known_failure_windows\n@assert_cwd_unchanged\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_add(src, path):\n\n    gr = GitRepo.clone(src, path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n    added = gr.add(filename)\n\n    eq_(added, {'success': True, 'file': filename})\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    # uncommitted:\n    ok_(gr.dirty)\n\n    filename = \"another.txt\"\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"Another file to add to git\")\n\n    # include committing:\n    added2 = gr.add(filename)\n    gr.commit(msg=\"Add two files.\")\n    eq_(added2, {'success': True, 'file': filename})\n\n    assert_in(filename, gr.get_indexed_files(),\n              \"%s not successfully added to %s\" % (filename, path))\n    ok_clean_git(path)\n\n\n@assert_cwd_unchanged\n@with_tree(tree={\n    'd': {'f1': 'content1',\n          'f2': 'content2'},\n    'file': 'content3',\n    'd2': {'f1': 'content1',\n          'f2': 'content2'},\n    'file2': 'content3'\n\n    })\ndef test_GitRepo_remove(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add('*')\n    gr.commit(\"committing all the files\")\n\n    eq_(gr.remove('file'), ['file'])\n    eq_(set(gr.remove('d', r=True, f=True)), {'d/f1', 'd/f2'})\n\n    eq_(set(gr.remove('*', r=True, f=True)), {'file2', 'd2/f1', 'd2/f2'})\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_GitRepo_commit(path):\n\n    gr = GitRepo(path)\n    filename = get_most_obscure_supported_name()\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"File to add to git\")\n\n    gr.add(filename)\n    gr.commit(\"Testing GitRepo.commit().\")\n    ok_clean_git(gr)\n    eq_(\"Testing GitRepo.commit().\",\n        gr.format_commit(\"%B\").strip())\n\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"changed content\")\n\n    gr.add(filename)\n    gr.commit(\"commit with options\", options=to_options(dry_run=True))\n    # wasn't actually committed:\n    ok_(gr.dirty)\n\n    # commit with empty message:\n    gr.commit()\n    ok_clean_git(gr)\n\n    # nothing to commit doesn't raise by default:\n    gr.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, gr.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(op.join(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, gr.commit, files=\"not-existing\")\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_indexed_files(src, path):\n\n    gr = GitRepo.clone(src, path)\n    idx_list = gr.get_indexed_files()\n\n    runner = Runner()\n    out = runner(['git', 'ls-files'], cwd=path)\n    out_list = list(filter(bool, out[0].split('\\n')))\n\n    for item in idx_list:\n        assert_in(item, out_list, \"%s not found in output of git ls-files in %s\" % (item, path))\n    for item in out_list:\n        assert_in(item, idx_list, \"%s not found in output of get_indexed_files in %s\" % (item, path))\n\n\n@with_tree([\n    ('empty', ''),\n    ('d1', (\n        ('empty', ''),\n        ('d2',\n            (('empty', ''),\n             )),\n        )),\n    ])\n@assert_cwd_unchanged(ok_to_chdir=True)\ndef test_normalize_path(git_path):\n\n    gr = GitRepo(git_path)\n\n    # cwd is currently outside the repo, so any relative path\n    # should be interpreted as relative to `annex_path`\n    assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, getpwd())\n\n    result = _normalize_path(gr.path, \"testfile\")\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # result = _normalize_path(gr.path, op.join('.', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    #\n    # result = _normalize_path(gr.path, op.join('testdir', '..', 'testfile'))\n    # eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n    # Note: By now, normpath within normalize_paths() is disabled, therefore\n    # disable these tests.\n\n    result = _normalize_path(gr.path, op.join('testdir', 'testfile'))\n    eq_(result, op.join(\"testdir\", \"testfile\"), \"_normalize_path() returned %s\" % result)\n\n    result = _normalize_path(gr.path, op.join(git_path, \"testfile\"))\n    eq_(result, \"testfile\", \"_normalize_path() returned %s\" % result)\n\n    # now we are inside, so\n    # OLD PHILOSOPHY: relative paths are relative to cwd and have\n    # to be converted to be relative to annex_path\n    # NEW PHILOSOPHY: still relative to repo! unless starts with . (curdir) or .. (pardir)\n    with chpwd(op.join(git_path, 'd1', 'd2')):\n\n        result = _normalize_path(gr.path, \"testfile\")\n        eq_(result, 'testfile', \"_normalize_path() returned %s\" % result)\n\n        # if not joined as directory name but just a prefix to the filename, should\n        # behave correctly\n        for d in (op.curdir, op.pardir):\n            result = _normalize_path(gr.path, d + \"testfile\")\n            eq_(result, d + 'testfile', \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.curdir, \"testfile\"))\n        eq_(result, op.join('d1', 'd2', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        result = _normalize_path(gr.path, op.join(op.pardir, 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n        assert_raises(FileNotInRepositoryError, _normalize_path, gr.path, op.join(git_path, '..', 'outside'))\n\n        result = _normalize_path(gr.path, op.join(git_path, 'd1', 'testfile'))\n        eq_(result, op.join('d1', 'testfile'), \"_normalize_path() returned %s\" % result)\n\n\ndef test_GitRepo_files_decorator():\n\n    class testclass(object):\n        def __init__(self):\n            self.path = op.join('some', 'where')\n\n        # TODO\n        # yoh:  logic is alien to me below why to have two since both look identical!\n        @normalize_paths\n        def decorated_many(self, files):\n            return files\n\n        @normalize_paths\n        def decorated_one(self, file_):\n            return file_\n\n    test_instance = testclass()\n\n    # When a single file passed -- single path returned\n    obscure_filename = get_most_obscure_supported_name()\n    file_to_test = op.join(test_instance.path, 'deep', obscure_filename)\n    # file doesn't exist\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = obscure_filename\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n    eq_(test_instance.decorated_one(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n\n    file_to_test = op.join(obscure_filename, 'beyond', 'obscure')\n    eq_(test_instance.decorated_many(file_to_test),\n                 _normalize_path(test_instance.path, file_to_test))\n\n    file_to_test = op.join(getpwd(), 'somewhere', 'else', obscure_filename)\n    assert_raises(FileNotInRepositoryError, test_instance.decorated_many,\n                  file_to_test)\n\n    # If a list passed -- list returned\n    files_to_test = ['now', op.join('a list', 'of'), 'paths']\n    expect = []\n    for item in files_to_test:\n        expect.append(_normalize_path(test_instance.path, item))\n    eq_(test_instance.decorated_many(files_to_test), expect)\n\n    eq_(test_instance.decorated_many(''), [])\n\n    assert_raises(ValueError, test_instance.decorated_many, 1)\n    assert_raises(ValueError, test_instance.decorated_one, 1)\n\n\n@skip_if_no_network\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_add(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    eq_(len(out), 1)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    out = gr.get_remotes()\n    assert_in('origin', out)\n    assert_in('github', out)\n    eq_(len(out), 2)\n    eq_('git://github.com/datalad/testrepo--basic--r1', gr.config['remote.github.url'])\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_remote_remove(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    gr.remove_remote('github')\n    out = gr.get_remotes()\n    eq_(len(out), 1)\n    assert_in('origin', out)\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:491\n@known_failure_windows\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_GitRepo_get_remote_url(orig_path, path):\n\n    gr = GitRepo.clone(orig_path, path)\n    gr.add_remote('github', 'git://github.com/datalad/testrepo--basic--r1')\n    eq_(gr.get_remote_url('origin'), orig_path)\n    eq_(gr.get_remote_url('github'),\n                 'git://github.com/datalad/testrepo--basic--r1')\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_pull(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n    clone.pull()\n    ok_(op.exists(op.join(clone_path, filename)))\n\n    # While at it, let's test _get_remotes_having_commit a bit\n    from datalad.distribution.get import _get_remotes_having_commit\n    clone.add_remote(\"very_origin\", test_path)\n    clone.fetch(\"very_origin\")\n    eq_(\n        _get_remotes_having_commit(clone, clone.get_hexsha()),\n        ['origin']\n    )\n    prev_commit = clone.get_hexsha('HEAD^')\n    eq_(\n        set(_get_remotes_having_commit(clone, prev_commit)),\n        {'origin', 'very_origin'}\n    )\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_fetch(test_path, orig_path, clone_path):\n\n    origin = GitRepo.clone(test_path, orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    origin.checkout(\"new_branch\", ['-b'])\n    with open(op.join(orig_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    origin.add(filename)\n    origin.commit(\"new file added.\")\n\n    fetched = clone.fetch(remote='origin')\n    # test FetchInfo list returned by fetch\n    eq_([u'origin/' + clone.get_active_branch(), u'origin/new_branch'],\n        [commit['ref'] for commit in fetched])\n\n    ok_clean_git(clone.path, annex=False)\n    assert_in(\"origin/new_branch\", clone.get_remote_branches())\n    assert_in(filename, clone.get_files(\"origin/new_branch\"))\n    assert_false(op.exists(op.join(clone_path, filename)))  # not checked out\n\n    # create a remote without an URL:\n    origin.add_remote('not-available', 'git://example.com/not/existing')\n    origin.config.unset('remote.not-available.url', where='local')\n\n    # fetch without provided URL\n    assert_raises(CommandError, origin.fetch, 'not-available')\n\n\ndef _path2localsshurl(path):\n    \"\"\"Helper to build valid localhost SSH urls on Windows too\"\"\"\n    from pathlib import Path\n    path = op.abspath(path)\n    p = Path(path)\n    if p.drive:\n        path = '/'.join(('/{}'.format(p.drive[0]),) + p.parts[1:])\n    url = \"ssh://localhost{}\".format(path)\n    return url\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_testrepos('.*basic.*', flavors=['local'])\n@with_tempfile\ndef test_GitRepo_ssh_fetch(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=False)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # we don't know any branches of the remote:\n    eq_([], repo.get_remote_branches())\n\n    fetched = repo.fetch(remote=\"ssh-remote\")\n    assert_in('ssh-remote/master', [commit['ref'] for commit in fetched])\n    ok_clean_git(repo)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually fetched it:\n    assert_in('ssh-remote/master', repo.get_remote_branches())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_pull(remote_path, repo_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify remote:\n    remote_repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(remote_repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    remote_repo.add(\"ssh_testfile.dat\")\n    remote_repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not locally known yet:\n    assert_not_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n    # pull changes:\n    repo.pull(remote=\"ssh-remote\", refspec=remote_repo.get_active_branch())\n    ok_clean_git(repo.path, annex=False)\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # we actually pulled the changes\n    assert_in(\"ssh_testfile.dat\", repo.get_indexed_files())\n\n\n# broken,possibly due to a GitPy issue with windows sshurls\n# see https://github.com/datalad/datalad/pull/3638\n@skip_if_on_windows\n@skip_ssh\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_ssh_push(repo_path, remote_path):\n    from datalad import ssh_manager\n\n    remote_repo = GitRepo(remote_path, create=True)\n    url = _path2localsshurl(remote_path)\n    socket_path = op.join(str(ssh_manager.socket_dir),\n                          get_connection_hash('localhost', bundled=True))\n    repo = GitRepo(repo_path, create=True)\n    repo.add_remote(\"ssh-remote\", url)\n\n    # modify local repo:\n    repo.checkout(\"ssh-test\", ['-b'])\n    with open(op.join(repo.path, \"ssh_testfile.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n    repo.add(\"ssh_testfile.dat\")\n    repo.commit(\"ssh_testfile.dat added.\")\n\n    # file is not known to the remote yet:\n    assert_not_in(\"ssh_testfile.dat\", remote_repo.get_indexed_files())\n\n    # push changes:\n    pushed = list(repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\"))\n    # test PushInfo\n    assert_in(\"refs/heads/ssh-test\", [p['from_ref'] for p in pushed])\n    assert_in(\"refs/heads/ssh-test\", [p['to_ref'] for p in pushed])\n\n    # the connection is known to the SSH manager, since fetch() requested it:\n    assert_in(socket_path, list(map(str, ssh_manager._connections)))\n    # and socket was created:\n    ok_(op.exists(socket_path))\n\n    # remote now knows the changes:\n    assert_in(\"ssh-test\", remote_repo.get_branches())\n    assert_in(\"ssh_testfile.dat\", remote_repo.get_files(\"ssh-test\"))\n\n    # amend to make it require \"--force\":\n    repo.commit(\"amended\", options=['--amend'])\n    # push without --force should yield an error:\n    res = repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\")\n    assert_in_results(\n        res,\n        from_ref='refs/heads/ssh-test',\n        to_ref='refs/heads/ssh-test',\n        operations=['rejected', 'error'],\n        note='[rejected] (non-fast-forward)',\n        remote='ssh-remote',\n    )\n    # now push using force:\n    repo.push(remote=\"ssh-remote\", refspec=\"ssh-test\", force=True)\n    # correct commit message in remote:\n    assert_in(\"amended\",\n              remote_repo.format_commit(\n                  '%s',\n                  list(remote_repo.get_branch_commits_('ssh-test'))[-1]\n              ))\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_push_n_checkout(orig_path, clone_path):\n\n    origin = GitRepo(orig_path)\n    clone = GitRepo.clone(orig_path, clone_path)\n    filename = get_most_obscure_supported_name()\n\n    with open(op.join(clone_path, filename), 'w') as f:\n        f.write(\"New file.\")\n    clone.add(filename)\n    clone.commit(\"new file added.\")\n    # TODO: need checkout first:\n    clone.push('origin', '+master:new-branch')\n    origin.checkout('new-branch')\n    ok_(op.exists(op.join(orig_path, filename)))\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_remote_update(path1, path2, path3):\n\n    git1 = GitRepo(path1)\n    git2 = GitRepo(path2)\n    git3 = GitRepo(path3)\n\n    git1.add_remote('git2', path2)\n    git1.add_remote('git3', path3)\n\n    # Setting up remote 'git2'\n    with open(op.join(path2, 'masterfile'), 'w') as f:\n        f.write(\"git2 in master\")\n    git2.add('masterfile')\n    git2.commit(\"Add something to master.\")\n    git2.checkout('branch2', ['-b'])\n    with open(op.join(path2, 'branch2file'), 'w') as f:\n        f.write(\"git2 in branch2\")\n    git2.add('branch2file')\n    git2.commit(\"Add something to branch2.\")\n\n    # Setting up remote 'git3'\n    with open(op.join(path3, 'masterfile'), 'w') as f:\n        f.write(\"git3 in master\")\n    git3.add('masterfile')\n    git3.commit(\"Add something to master.\")\n    git3.checkout('branch3', ['-b'])\n    with open(op.join(path3, 'branch3file'), 'w') as f:\n        f.write(\"git3 in branch3\")\n    git3.add('branch3file')\n    git3.commit(\"Add something to branch3.\")\n\n    git1.update_remote()\n\n    # checkouts are 'tests' themselves, since they'll raise CommandError\n    # if something went wrong\n    git1.checkout('branch2')\n    git1.checkout('branch3')\n\n    branches1 = git1.get_branches()\n    eq_({'branch2', 'branch3'}, set(branches1))\n\n\n# TODO: Why was it \"flavors=local_testrepo_flavors\" ? What's the windows issue here?\n@with_testrepos('.*git.*', flavors=['clone'])\n@with_tempfile\ndef test_GitRepo_get_files(url, path):\n\n    gr = GitRepo.clone(url, path)\n\n    # get the expected files via os for comparison:\n    os_files = set()\n    for (dirpath, dirnames, filenames) in os.walk(path):\n        rel_dir = os.path.relpath(dirpath, start=path)\n        if rel_dir.startswith(\".git\"):\n            continue\n        for file_ in filenames:\n            file_path = os.path.normpath(op.join(rel_dir, file_))\n            os_files.add(file_path)\n\n    # get the files via GitRepo:\n    local_files = set(gr.get_files())\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n\n    eq_(local_files, set(gr.get_indexed_files()))\n    eq_(local_files, remote_files)\n    eq_(local_files, os_files)\n\n    # create a different branch:\n    gr.checkout('new_branch', ['-b'])\n    filename = 'another_file.dat'\n    with open(op.join(path, filename), 'w') as f:\n        f.write(\"something\")\n    gr.add(filename)\n    gr.commit(\"Added.\")\n\n    # now get the files again:\n    local_files = set(gr.get_files())\n    eq_(local_files, os_files.union({filename}))\n    # retrieve remote branch again, which should not have changed:\n    remote_files = set(gr.get_files(branch=\"origin/master\"))\n    eq_(remote_files, os_files)\n    eq_(set([filename]), local_files.difference(remote_files))\n\n    # switch back and query non-active branch:\n    gr.checkout('master')\n    local_files = set(gr.get_files())\n    branch_files = set(gr.get_files(branch=\"new_branch\"))\n    eq_(set([filename]), branch_files.difference(local_files))\n\n\n# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:505\n@known_failure_windows\n@with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_GitRepo_get_toppath(repo, tempdir, repo2):\n    reporeal = op.realpath(repo)\n    eq_(GitRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(GitRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    GitRepo(repo2, create=True)\n    repo2real = op.realpath(repo2)\n    nested = op.join(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(GitRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(GitRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(GitRepo.get_toppath(tempdir), None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_dirty(path):\n\n    repo = GitRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # untracked file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt')\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(not repo.dirty)\n    # modified file\n    with open(op.join(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt')\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # An empty directory doesn't count as dirty.\n    os.mkdir(op.join(path, \"empty\"))\n    ok_(not repo.dirty)\n    # Neither does an empty directory with an otherwise empty directory.\n    os.mkdir(op.join(path, \"empty\", \"empty-again\"))\n    ok_(not repo.dirty)\n\n    subm = GitRepo(repo.pathobj / \"subm\", create=True)\n    (subm.pathobj / \"subfile\").write_text(u\"\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n    (subm.pathobj / \"subfile\").write_text(u\"changed\")\n    ok_(repo.dirty)\n\n    # User configuration doesn't affect .dirty's answer.\n    repo.config.set(\"diff.ignoreSubmodules\", \"all\", where=\"local\")\n    ok_(repo.dirty)\n    # GitRepo.commit currently can't handle this setting, so remove it for the\n    # save() calls below.\n    repo.config.unset(\"diff.ignoreSubmodules\", where=\"local\")\n    subm.save()\n    repo.save()\n    ok_(not repo.dirty)\n\n    repo.config.set(\"status.showUntrackedFiles\", \"no\", where=\"local\")\n    create_tree(repo.path, {\"untracked_dir\": {\"a\": \"a\"}})\n    ok_(repo.dirty)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_merge_base(src):\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    assert_raises(ValueError, repo.get_merge_base, [])\n    branch1 = repo.get_active_branch()\n    branch1_hexsha = repo.get_hexsha()\n    eq_(len(branch1_hexsha), 40)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # Let's create a detached branch\n    branch2 = \"_detach_\"\n    repo.checkout(branch2, options=[\"--orphan\"])\n    # it will have all the files\n    # Must not do:  https://github.com/gitpython-developers/GitPython/issues/375\n    # repo.git_add('.')\n    repo.add('*')\n    # NOTE: fun part is that we should have at least a different commit message\n    # so it results in a different checksum ;)\n    repo.commit(\"committing again\")\n    assert(repo.get_indexed_files())  # we did commit\n    assert(repo.get_merge_base(branch1) is None)\n    assert(repo.get_merge_base([branch2, branch1]) is None)\n\n    # Let's merge them up -- then merge base should match the master\n    repo.merge(branch1, allow_unrelated=True)\n    eq_(repo.get_merge_base(branch1), branch1_hexsha)\n\n    # if points to some empty/non-existing branch - should also be None\n    assert(repo.get_merge_base(['nonexistent', branch2]) is None)\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_git_get_branch_commits_(src):\n\n    repo = GitRepo(src, create=True)\n    with open(op.join(src, 'file.txt'), 'w') as f:\n        f.write('load')\n    repo.add('*')\n    repo.commit('committing')\n\n    commits_default = list(repo.get_branch_commits_())\n    commits = list(repo.get_branch_commits_('master'))\n    eq_(commits, commits_default)\n    eq_(len(commits), 1)\n\n\n@with_testrepos(flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_get_tracking_branch(o_path, c_path):\n\n    clone = GitRepo.clone(o_path, c_path)\n    # Note, that the default branch might differ even if it is always 'master'.\n    # For direct mode annex repositories it would then be \"annex/direct/master\"\n    # for example. Therefore use whatever branch is checked out by default:\n    master_branch = clone.get_active_branch()\n    ok_(master_branch)\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n\n    clone.checkout('new_branch', ['-b'])\n\n    eq_((None, None), clone.get_tracking_branch())\n\n    eq_(('origin', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch(master_branch))\n\n    clone.checkout(master_branch, options=[\"--track\", \"-btopic\"])\n    eq_(('.', 'refs/heads/' + master_branch),\n        clone.get_tracking_branch())\n    eq_((None, None),\n        clone.get_tracking_branch(remote_only=True))\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_submodule_deinit(path):\n    from datalad.support.annexrepo import AnnexRepo\n\n    top_repo = AnnexRepo(path, create=False)\n    eq_({'subm 1', '2'}, {s.name for s in top_repo.get_submodules()})\n    # note: here init=True is ok, since we are using it just for testing\n    with swallow_logs(new_level=logging.WARN) as cml:\n        top_repo.update_submodule('subm 1', init=True)\n        assert_in('Do not use update_submodule with init=True', cml.out)\n    top_repo.update_submodule('2', init=True)\n\n    # ok_(all([s.module_exists() for s in top_repo.get_submodules()]))\n    # TODO: old assertion above if non-bare? (can't use \"direct mode\" in test_gitrepo)\n    # Alternatively: New testrepo (plain git submodules) and have a dedicated\n    # test for annexes in addition\n    ok_(all([GitRepo.is_valid_repo(op.join(top_repo.path, s.path))\n             for s in top_repo.get_submodules()]))\n\n    # modify submodule:\n    with open(op.join(top_repo.path, 'subm 1', 'file_ut.dat'), \"w\") as f:\n        f.write(\"some content\")\n\n    assert_raises(CommandError, top_repo.deinit_submodule, 'sub1')\n\n    # using force should work:\n    top_repo.deinit_submodule('subm 1', force=True)\n\n    ok_(not top_repo.repo.submodule('subm 1').module_exists())\n\n\n@with_testrepos(\".*basic_git.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_GitRepo_add_submodule(source, path):\n\n    top_repo = GitRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n    ok_clean_git(path)\n    ok_clean_git(op.join(path, 'sub'))\n\n\ndef test_GitRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef check_update_submodule_init_adjust_branch(is_ancestor, path):\n    src = GitRepo(op.join(path, \"src\"), create=True)\n    src_sub = GitRepo(op.join(src.path, \"sub\"), create=True)\n    src_sub.commit(msg=\"c0\", options=[\"--allow-empty\"])\n    src_sub.commit(msg=\"c1\", options=[\"--allow-empty\"])\n    src.add_submodule('sub', name='sub')\n    src.commit(msg=\"Add submodule\")\n\n    # Move subdataset past the registered commit...\n    hexsha_registered = src_sub.get_hexsha()\n    if is_ancestor:\n        # ... where the registered commit is an ancestor of the new one.\n        src_sub.commit(msg=\"c2\", options=[\"--allow-empty\"])\n    else:\n        # ... where the registered commit is NOT an ancestor of the new one.\n        src_sub.call_git([\"reset\", \"--hard\", \"master~1\"])  # c0\n    hexsha_sub = src_sub.get_hexsha()\n\n    clone = GitRepo.clone(url=src.path,\n                          path=op.join(path, \"clone\"),\n                          create=True)\n    clone_sub = GitRepo.clone(url=src_sub.path,\n                              path=op.join(clone.path, \"sub\"),\n                              create=True)\n    ok_(clone.dirty)\n    eq_(clone_sub.get_active_branch(), \"master\")\n    eq_(hexsha_sub, clone_sub.get_hexsha())\n\n    clone.update_submodule(\"sub\", init=True)\n\n    assert_false(clone.dirty)\n    eq_(hexsha_registered, clone_sub.get_hexsha())\n    if is_ancestor:\n        eq_(clone_sub.get_active_branch(), \"master\")\n    else:\n        assert_false(clone_sub.get_active_branch())\n\n\ndef test_GitRepo_update_submodule_init_adjust_branch():\n    yield check_update_submodule_init_adjust_branch, True\n    yield check_update_submodule_init_adjust_branch, False\n\n\ndef test_GitRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile\ndef test_get_submodules_parent_on_unborn_branch(path):\n    repo = GitRepo(path, create=True)\n    subrepo = GitRepo(op.join(path, \"sub\"), create=True)\n    subrepo.commit(msg=\"s\", options=[\"--allow-empty\"])\n    repo.add_submodule(path=\"sub\")\n    eq_([s.name for s in repo.get_submodules()],\n        [\"sub\"])\n\n\ndef test_to_options():\n\n    class Some(object):\n\n        def cmd_func(self, git_options=None, annex_options=None, options=None):\n\n            git_options = git_options[:] if git_options else []\n            annex_options = annex_options[:] if annex_options else []\n            options = options[:] if options else []\n\n            faked_cmd_call = ['git'] + git_options + ['annex'] + \\\n                             annex_options + ['my_cmd'] + options\n\n            return faked_cmd_call\n\n    eq_(Some().cmd_func(options=to_options(m=\"bla\", force=True)),\n        ['git', 'annex', 'my_cmd', '--force', '-m', 'bla'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\"),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C', '/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n    eq_(Some().cmd_func(git_options=to_options(C=\"/some/where\", split_single_char_options=False),\n                        annex_options=to_options(JSON=True),\n                        options=to_options(unused=True)),\n        ['git', '-C/some/where', 'annex', '--JSON', 'my_cmd', '--unused'])\n\n\ndef test_to_options_from_gitpython():\n    \"\"\"Imported from GitPython and modified.\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    eq_([\"-s\"], to_options(**{'s': True}))\n    eq_([\"-s\", \"5\"], to_options(**{'s': 5}))\n    eq_([], to_options(**{'s': None}))\n\n    eq_([\"--max-count\"], to_options(**{'max_count': True}))\n    eq_([\"--max-count=5\"], to_options(**{'max_count': 5}))\n    eq_([\"--max-count=0\"], to_options(**{'max_count': 0}))\n    eq_([], to_options(**{'max_count': None}))\n\n    # Multiple args are supported by using lists/tuples\n    eq_([\"-L\", \"1-3\", \"-L\", \"12-18\"], to_options(**{'L': ('1-3', '12-18')}))\n    eq_([\"-C\", \"-C\"], to_options(**{'C': [True, True, None, False]}))\n\n    # order is undefined\n    res = to_options(**{'s': True, 't': True})\n    eq_({'-s', '-t'}, set(res))\n\n\n@with_tempfile\ndef test_GitRepo_count_objects(repo_path):\n\n    repo = GitRepo(repo_path, create=True)\n    # test if dictionary returned\n    eq_(isinstance(repo.count_objects, dict), True)\n    # test if dictionary contains keys and values we expect\n    empty_count = {'count': 0, 'garbage': 0,  'in-pack': 0, 'packs': 0, 'prune-packable': 0,\n                   'size': 0, 'size-garbage': 0, 'size-pack': 0}\n    eq_(empty_count, repo.count_objects)\n\n\n# this is simply broken on win, but less important\n# https://github.com/datalad/datalad/issues/3639\n@skip_if_on_windows\n@with_tempfile\ndef test_optimized_cloning(path):\n    # make test repo with one file and one commit\n    originpath = op.join(path, 'origin')\n    repo = GitRepo(originpath, create=True)\n    with open(op.join(originpath, 'test'), 'w') as f:\n        f.write('some')\n    repo.add('test')\n    repo.commit('init')\n    ok_clean_git(originpath, annex=False)\n    from glob import glob\n\n    def _get_inodes(repo):\n        return dict(\n            [(os.path.join(*o.split(os.sep)[-2:]),\n              os.stat(o).st_ino)\n             for o in glob(os.path.join(repo.path,\n                                        repo.get_git_dir(repo),\n                                        'objects', '*', '*'))])\n\n    origin_inodes = _get_inodes(repo)\n    # now clone it in different ways and see what happens to the object storage\n    from datalad.support.network import get_local_file_url\n    clonepath = op.join(path, 'clone')\n    for src in (originpath, get_local_file_url(originpath, compatibility='git')):\n        # deprecated\n        assert_raises(DeprecatedError, GitRepo, url=src, path=clonepath)\n        clone = GitRepo.clone(url=src, path=clonepath, create=True)\n        clone_inodes = _get_inodes(clone)\n        eq_(origin_inodes, clone_inodes, msg='with src={}'.format(src))\n        rmtree(clonepath)\n#        del clone\n#        gc.collect()\n        # Note: del needed, since otherwise WeakSingletonRepo would just\n        # return the original object in second run\n\n\n@with_tempfile\n@with_tempfile\ndef test_GitRepo_gitpy_injection(path, path2):\n\n    gr = GitRepo(path, create=True)\n    gr._GIT_COMMON_OPTIONS.extend(['test-option'])\n\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # once set, these option should be persistent across git calls:\n    with assert_raises(GitCommandError) as cme:\n        gr.repo.git.another_unknown_git_command()\n    assert_in('test-option', exc_str(cme.exception))\n\n    # but other repos should not be affected:\n    gr2 = GitRepo(path2, create=True)\n    with assert_raises(GitCommandError) as cme:\n        gr2.repo.git.unknown_git_command()\n    assert_not_in('test-option', exc_str(cme.exception))\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_GitRepo_flyweight(path1, path2):\n\n    repo1 = GitRepo(path1, create=True)\n    assert_is_instance(repo1, GitRepo)\n    # instantiate again:\n    repo2 = GitRepo(path1, create=False)\n    assert_is_instance(repo2, GitRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in a different way:\n    with chpwd(path1):\n        repo3 = GitRepo(op.relpath(path1, start=path2), create=False)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # and realpath attribute is the same, so they are still equal:\n    ok_(repo1 == repo3)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_GitRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(GitRepo, path, store)\n\n\n@with_tree(tree={'ignore-sub.me': {'a_file.txt': 'some content'},\n                 'ignore.me': 'ignored content',\n                 'dontigno.re': 'other content'})\ndef test_GitRepo_gitignore(path):\n\n    gr = GitRepo(path, create=True)\n    sub = GitRepo(op.join(path, 'ignore-sub.me'))\n    # we need to commit something, otherwise add_submodule\n    # will already refuse the submodule for having no commit\n    sub.add('a_file.txt')\n    sub.commit()\n\n    from datalad.support.exceptions import GitIgnoreError\n\n    with open(op.join(path, '.gitignore'), \"w\") as f:\n        f.write(\"*.me\")\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add('ignore.me')\n    eq_(cme.exception.paths, ['ignore.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add_submodule(path='ignore-sub.me')\n    eq_(cme.exception.paths, ['ignore-sub.me'])\n\n    with assert_raises(GitIgnoreError) as cme:\n        gr.add(['ignore.me', 'dontigno.re', op.join('ignore-sub.me', 'a_file.txt')])\n    eq_(set(cme.exception.paths), {'ignore.me', 'ignore-sub.me'})\n\n    eq_(gr.get_gitattributes('.')['.'], {})  # nothing is recorded within .gitattributes\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_set_remote_url(path):\n\n    gr = GitRepo(path, create=True)\n    gr.add_remote('some', 'http://example.com/.git')\n    eq_(gr.config['remote.some.url'],\n                 'http://example.com/.git')\n    # change url:\n    gr.set_remote_url('some', 'http://believe.it')\n    eq_(gr.config['remote.some.url'],\n                 'http://believe.it')\n\n    # set push url:\n    gr.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    eq_(gr.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n\n    # add remote without url\n    url2 = 'http://repo2.example.com/.git'\n    gr.add_remote('some-without-url', url2)\n    eq_(gr.config['remote.some-without-url.url'], url2)\n    # \"remove\" it\n    gr.config.unset('remote.some-without-url.url', where='local')\n    with assert_raises(KeyError):\n        gr.config['remote.some-without-url.url']\n    eq_(set(gr.get_remotes()), {'some', 'some-without-url'})\n    eq_(set(gr.get_remotes(with_urls_only=True)), {'some'})\n\n\n@with_tempfile(mkdir=True)\ndef test_gitattributes(path):\n    gr = GitRepo(path, create=True)\n    # starts without any attributes file\n    ok_(not op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {})\n    # bool is a tag or unsets, anything else is key/value\n    gr.set_gitattributes([('*', {'tag': True}), ('*', {'sec.key': 'val'})])\n    ok_(op.exists(op.join(gr.path, '.gitattributes')))\n    eq_(gr.get_gitattributes('.')['.'], {'tag': True, 'sec.key': 'val'})\n    # unset by amending the record, but does not remove notion of the\n    # tag entirely\n    gr.set_gitattributes([('*', {'tag': False})])\n    eq_(gr.get_gitattributes('.')['.'], {'tag': False, 'sec.key': 'val'})\n    # attributes file is not added or commited, we can ignore such\n    # attributes\n    eq_(gr.get_gitattributes('.', index_only=True)['.'], {})\n\n    # we can send absolute path patterns and write to any file, and\n    # the patterns will be translated relative to the target file\n    gr.set_gitattributes([\n        (op.join(gr.path, 'relative', 'ikethemike/**'), {'bang': True})],\n        attrfile=op.join('relative', '.gitattributes'))\n    # directory and file get created\n    ok_(op.exists(op.join(gr.path, 'relative', '.gitattributes')))\n    eq_(gr.get_gitattributes(\n        op.join(gr.path, 'relative', 'ikethemike', 'probe')),\n        # always comes out relative to the repo root, even if abs goes in\n        {op.join('relative', 'ikethemike', 'probe'):\n            {'tag': False, 'sec.key': 'val', 'bang': True}})\n    if get_encoding_info()['default'] != 'ascii' and not on_windows:\n        # do not perform this on obscure systems without anything like UTF\n        # it is not relevant whether a path actually exists, and paths\n        # with spaces and other funky stuff are just fine\n        funky = u'{} {}'.format(\n            get_most_obscure_supported_name(),\n            get_most_obscure_supported_name())\n        gr.set_gitattributes([(funky, {'this': 'that'})])\n        eq_(gr.get_gitattributes(funky)[funky], {\n            'this': 'that',\n            'tag': False,\n            'sec.key': 'val',\n        })\n\n    # mode='w' should replace the entire file:\n    gr.set_gitattributes([('**', {'some': 'nonsense'})], mode='w')\n    eq_(gr.get_gitattributes('.')['.'], {'some': 'nonsense'})\n\n\n@with_tempfile(mkdir=True)\ndef test_get_hexsha_tag(path):\n    gr = GitRepo(path, create=True)\n    gr.commit(msg=\"msg\", options=[\"--allow-empty\"])\n    gr.tag(\"atag\", message=\"atag msg\")\n    # get_hexsha() dereferences a tag to a commit.\n    eq_(gr.get_hexsha(\"atag\"), gr.get_hexsha())\n\n\n@with_tempfile(mkdir=True)\ndef test_get_tags(path):\n    from unittest.mock import patch\n\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_tags(), [])\n    eq_(gr.describe(), None)\n\n    # Explicitly override the committer date because tests may set it to a\n    # fixed value, but we want to check that the returned tags are sorted by\n    # the date the tag (for annotaged tags) or commit (for lightweight tags)\n    # was created.\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Thu, 07 Apr 2005 22:13:13 +0200\"}):\n        create_tree(gr.path, {'file': \"\"})\n        gr.add('file')\n        gr.commit(msg=\"msg\")\n        eq_(gr.get_tags(), [])\n        eq_(gr.describe(), None)\n\n        gr.tag(\"nonannotated\")\n        tags1 = [{'name': 'nonannotated', 'hexsha': gr.get_hexsha()}]\n        eq_(gr.get_tags(), tags1)\n        eq_(gr.describe(), None)\n        eq_(gr.describe(tags=True), tags1[0]['name'])\n\n    first_commit = gr.get_hexsha()\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 08 Apr 2005 22:13:13 +0200\"}):\n\n        create_tree(gr.path, {'file': \"123\"})\n        gr.add('file')\n        gr.commit(msg=\"changed\")\n\n    with patch.dict(\"os.environ\", {\"GIT_COMMITTER_DATE\":\n                                   \"Fri, 09 Apr 2005 22:13:13 +0200\"}):\n        gr.tag(\"annotated\", message=\"annotation\")\n    # The annotated tag happened later, so it comes last.\n    tags2 = tags1 + [{'name': 'annotated', 'hexsha': gr.get_hexsha()}]\n    eq_(gr.get_tags(), tags2)\n    eq_(gr.describe(), tags2[1]['name'])\n\n    # compare prev commit\n    eq_(gr.describe(commitish=first_commit), None)\n    eq_(gr.describe(commitish=first_commit, tags=True), tags1[0]['name'])\n\n    gr.tag('specific', commit='HEAD~1')\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD~1'))\n    assert_in('specific', gr.get_tags(output='name'))\n\n    # retag a different commit\n    assert_raises(CommandError, gr.tag, 'specific', commit='HEAD')\n    # force it\n    gr.tag('specific', commit='HEAD', options=['-f'])\n    eq_(gr.get_hexsha('specific'), gr.get_hexsha('HEAD'))\n\n    # delete\n    gr.call_git(['tag', '-d', 'specific'])\n    eq_(gr.get_tags(), tags2)\n    # more than one\n    gr.tag('one')\n    gr.tag('two')\n    gr.call_git(['tag', '-d', 'one', 'two'])\n    eq_(gr.get_tags(), tags2)\n\n\n@with_tree(tree={'1': \"\"})\ndef test_get_commit_date(path):\n    gr = GitRepo(path, create=True)\n    eq_(gr.get_commit_date(), None)\n\n    # Let's make a commit with a custom date\n    DATE = \"Wed Mar 14 03:47:30 2018 -0000\"\n    DATE_EPOCH = 1520999250\n    gr.add('1')\n    gr.commit(\"committed\", date=DATE)\n    gr = GitRepo(path, create=True)\n    date = gr.get_commit_date()\n    neq_(date, None)\n    eq_(date, DATE_EPOCH)\n\n    eq_(date, gr.get_commit_date('master'))\n    # and even if we get into a detached head\n    gr.checkout(gr.get_hexsha())\n    eq_(gr.get_active_branch(), None)\n    eq_(date, gr.get_commit_date('master'))\n\n\n@with_tree(tree={\"foo\": \"foo content\",\n                 \"bar\": \"bar content\"})\ndef test_fake_dates(path):\n    gr = GitRepo(path, create=True, fake_dates=True)\n\n    gr.add(\"foo\")\n    gr.commit(\"commit foo\")\n\n    seconds_initial = gr.config.obtain(\"datalad.fake-dates-start\")\n\n    # First commit is incremented by 1 second.\n    eq_(seconds_initial + 1, gr.get_commit_date())\n\n    # The second commit by 2.\n    gr.add(\"bar\")\n    gr.commit(\"commit bar\")\n    eq_(seconds_initial + 2, gr.get_commit_date())\n\n    # If we checkout another branch, its time is still based on the latest\n    # timestamp in any local branch.\n    gr.checkout(\"other\", options=[\"--orphan\"])\n    with open(op.join(path, \"baz\"), \"w\") as ofh:\n        ofh.write(\"baz content\")\n    gr.add(\"baz\")\n    gr.commit(\"commit baz\")\n    eq_(gr.get_active_branch(), \"other\")\n    eq_(seconds_initial + 3, gr.get_commit_date())\n\n\ndef test_guard_BadName():\n    from gitdb.exc import BadName\n\n    calls = []\n\n    class Vulnerable(object):\n        def precommit(self):\n            calls.append('precommit')\n\n        @guard_BadName\n        def __call__(self, x, y=2):\n            if not calls:\n                calls.append(1)\n                raise BadName\n            return x+y\n    v = Vulnerable()\n    eq_(v(1, y=3), 4)\n    eq_(calls, [1, 'precommit'])\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\ndef test_custom_runner_protocol(path):\n    # Check that a runner with a non-default protocol gets wired up correctly.\n    prot = ExecutionTimeProtocol()\n    gr = GitRepo(path, runner=Runner(cwd=path, protocol=prot), create=True)\n\n    ok_(len(prot) > 0)\n    ok_(prot[0]['duration'] >= 0)\n\n    def check(prev_len, prot, command):\n        # Check that the list grew and has the expected command without\n        # assuming that it gained _only_ a one command.\n        ok_(len(prot) > prev_len)\n        assert_in(command,\n                  sum([p[\"command\"] for p in prot[prev_len:]], []))\n\n    prev_len = len(prot)\n    gr.add(\"foo\")\n    check(prev_len, prot, \"add\")\n\n    # commit no longer uses a Runner with protocol capabilities\n    #prev_len = len(prot)\n    #gr.commit(\"commit foo\")\n    #check(prev_len, prot, \"commit\")\n\n    ok_(all(p['duration'] >= 0 for p in prot))\n\n\n@with_tempfile(mkdir=True)\ndef test_duecredit(path):\n    # Just to check that no obvious side-effects\n    run = Runner(cwd=path).run\n    cmd = [\n        sys.executable, \"-c\",\n        \"from datalad.support.gitrepo import GitRepo; GitRepo(%r, create=True)\" % path\n    ]\n\n    env = os.environ.copy()\n\n    # Test with duecredit not enabled for sure\n    env.pop('DUECREDIT_ENABLE', None)\n    # Alternative workaround for what to be fixed by\n    # https://github.com/datalad/datalad/pull/3215\n    # where underlying datalad process might issue a warning since our temp\n    # cwd is not matching possibly present PWD env variable\n    env.pop('PWD', None)\n\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err  # Let's not depend on where duecredit decides to spit out\n    # All quiet\n    eq_(outs, '')\n\n    # and now enable DUECREDIT - output could come to stderr\n    env['DUECREDIT_ENABLE'] = '1'\n    out, err = run(cmd, env=env, expect_stderr=True)\n    outs = out + err\n\n    if external_versions['duecredit']:\n        assert_in('Data management and distribution platform', outs)\n    else:\n        eq_(outs, '')\n\n\n@with_tempfile(mkdir=True)\ndef test_GitRepo_get_revisions(path):\n    gr = GitRepo(path, create=True)\n\n    def commit(msg):\n        gr.commit(msg=msg, options=[\"--allow-empty\"])\n\n    # We catch the error and return empty if the current branch doesn't have a\n    # commit checked out.\n    eq_(gr.get_revisions(), [])\n\n    # But will raise if on a bad ref name, including an unborn branch.\n    with assert_raises(CommandError):\n        gr.get_revisions(\"master\")\n\n    # By default, we query HEAD.\n    commit(\"1\")\n    eq_(len(gr.get_revisions()), 1)\n\n    gr.checkout(\"other\", options=[\"-b\"])\n    commit(\"2\")\n\n    # We can also query branch by name.\n    eq_(len(gr.get_revisions(\"master\")), 1)\n    eq_(len(gr.get_revisions(\"other\")), 2)\n\n    # \"name\" is sugar for [\"name\"].\n    eq_(gr.get_revisions(\"master\"),\n        gr.get_revisions([\"master\"]))\n\n    gr.checkout(\"master\")\n    commit(\"3\")\n    eq_(len(gr.get_revisions(\"master\")), 2)\n    # We can pass multiple revisions...\n    eq_(len(gr.get_revisions([\"master\", \"other\"])), 3)\n    # ... or options like --all and --branches\n    eq_(gr.get_revisions([\"master\", \"other\"]),\n        gr.get_revisions(options=[\"--all\"]))\n\n    # Ranges are supported.\n    eq_(gr.get_revisions(\"master..\"), [])\n\n\n@with_tree({\"foo\": \"foo\"})\ndef test_gitrepo_add_to_git_with_annex_v7(path):\n    from datalad.support.annexrepo import AnnexRepo\n    ar = AnnexRepo(path, create=True, version=7)\n    gr = GitRepo(path)\n    gr.add(\"foo\")\n    gr.commit(msg=\"c1\")\n    assert_false(ar.is_under_annex(\"foo\"))\n\n\n@with_tree({\"foo\": \"foo\", \"bar\": \"bar\"})\ndef test_gitrepo_call_git_methods(path):\n    gr = GitRepo(path)\n    gr.add([\"foo\", \"bar\"])\n    gr.commit(msg=\"foobar\")\n    gr.call_git([\"mv\"], files=[\"foo\", \"foo.txt\"])\n    ok_(op.exists(op.join(gr.path, 'foo.txt')))\n\n    for expect_fail, check in [(False, assert_in),\n                               (True, assert_not_in)]:\n        with swallow_logs(new_level=logging.DEBUG) as cml:\n            with assert_raises(CommandError):\n                gr.call_git([\"mv\"], files=[\"notthere\", \"dest\"],\n                            expect_fail=expect_fail)\n            check(\"notthere\", cml.out)\n\n    eq_(list(gr.call_git_items_([\"ls-files\"])),\n        [\"bar\", \"foo.txt\"])\n    eq_(list(gr.call_git_items_([\"ls-files\", \"-z\"], sep=\"\\0\")),\n        # Note: The custom separator has trailing empty item, but this is an\n        # arbitrary command with unknown output it isn't safe to trim it.\n        [\"bar\", \"foo.txt\", \"\"])\n\n    with assert_raises(AssertionError):\n        gr.call_git_oneline([\"ls-files\"])\n\n    eq_(gr.call_git_oneline([\"ls-files\"], files=[\"bar\"]),\n        \"bar\")\n\n    ok_(gr.call_git_success([\"rev-parse\", \"HEAD^{commit}\"]))\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        assert_false(gr.call_git_success([\"rev-parse\", \"HEAD^{blob}\"]))\n        assert_not_in(\"blob\", cml.out)\n\n\n@skip_if_no_network\n@with_tempfile\ndef _test_protocols(proto, destdir):\n    GitRepo.clone('%s://github.com/datalad-tester/testtt' % proto, destdir)\n\n\n@integration\ndef test_protocols():\n    # git-annex-standalone build can get git bundle which would fail to\n    # download via https, resulting in messages such as\n    #  fatal: unable to find remote helper for 'https'\n    # which happened with git-annex-standalone 7.20191017+git2-g7b13db551-1~ndall+1\n\n    # http is well tested already\n    for proto in 'git', 'https':\n        yield _test_protocols, proto\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\nDatalad version: 0.12.2.dev239\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.9\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 168ms\nInstalled 25 packages in 5.92s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nResolved 65 packages in 5.09s\nPrepared 1 package in 3.48s\nInstalled 59 packages in 263ms\n + appdirs==1.4.4\n + backports-tarfile==1.2.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.17.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + cryptography==44.0.0\n + datalad==0.12.2 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133)\n + deprecated==1.2.15\n + distro==1.9.0\n + duecredit==0.10.2\n + exifread==3.0.0\n + fasteners==0.19\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.11.0\n + idna==3.10\n + importlib-metadata==8.5.0\n + iso8601==2.1.0\n + jaraco-classes==3.4.0\n + jaraco-context==6.0.1\n + jaraco-functools==4.1.0\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==25.6.0\n + keyrings-alt==5.0.2\n + looseversion==1.3.0\n + lxml==5.3.0\n + more-itertools==10.5.0\n + msgpack==1.1.0\n + multidict==6.1.0\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.15.0\n + pillow==11.0.0\n + propcache==0.2.1\n + pycparser==2.22\n + pygithub==2.5.0\n + pyjwt==2.10.1\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.2\n + requests==2.32.3\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.6\n + tqdm==4.67.1\n + urllib3==1.26.20\n + vcrpy==7.0.0\n + whoosh==2.7.4\n + wrapt==1.17.0\n + yarl==1.18.3\nResolved 99 packages in 38ms\nInstalled 32 packages in 684ms\n + alabaster==0.7.16\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.16.0\n + build==1.2.2.post1\n + distlib==0.3.9\n + docutils==0.21.2\n + filelock==3.16.1\n + gprof2dot==2024.6.6\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.10.0\n + markupsafe==3.0.2\n + nose-timer==1.0.1\n + platformdirs==4.3.6\n + psutil==6.1.1\n + pygments==2.18.0\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==7.4.7\n + sphinx-rtd-theme==3.0.2\n + sphinxcontrib-applehelp==2.0.0\n + sphinxcontrib-devhelp==2.0.0\n + sphinxcontrib-htmlhelp==2.1.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==2.0.0\n + sphinxcontrib-serializinghtml==2.0.0\n + tabulate==0.9.0\n + virtualenv==20.28.0\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133\nplugins: cov-6.0.0\ncollected 56 items\n\nr2e_tests/test_1.py FEFEFFEEFFEE.EEEEEEEEEEEEFEEEFEsxsF..EFEEEFFFFFFF.FF [ 92%]\nFFFx                                                                     [100%]\n\n==================================== ERRORS ====================================\n______________ ERROR at setup of test_GitRepo_instance_from_clone ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 97\n  @assert_cwd_unchanged\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_instance_from_clone(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:97\n__________ ERROR at setup of test_GitRepo_instance_from_not_existing ___________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 126\n  @assert_cwd_unchanged\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_instance_from_not_existing(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:126\n____________________ ERROR at setup of test_GitRepo_equals _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 186\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_equals(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:186\n______________________ ERROR at setup of test_GitRepo_add ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 200\n  @known_failure_windows\n  @assert_cwd_unchanged\n  @with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_add(src, path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:200\n_______________ ERROR at setup of test_GitRepo_get_indexed_files _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 294\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_get_indexed_files(src, path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:294\n____________________ ERROR at setup of test_normalize_path _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 311\n  @with_tree([\n      ('empty', ''),\n      ('d1', (\n          ('empty', ''),\n          ('d2',\n              (('empty', ''),\n               )),\n          )),\n      ])\n  @assert_cwd_unchanged(ok_to_chdir=True)\n  def test_normalize_path(git_path):\nE       fixture 'git_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:311\n__________________ ERROR at setup of test_GitRepo_remote_add ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 428\n  @skip_if_no_network\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_remote_add(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:428\n_________________ ERROR at setup of test_GitRepo_remote_remove _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 445\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_remote_remove(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:445\n________________ ERROR at setup of test_GitRepo_get_remote_url _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 458\n  @known_failure_windows\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_get_remote_url(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:458\n_____________________ ERROR at setup of test_GitRepo_pull ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 470\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_pull(test_path, orig_path, clone_path):\nE       fixture 'test_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:470\n_____________________ ERROR at setup of test_GitRepo_fetch _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 501\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_fetch(test_path, orig_path, clone_path):\nE       fixture 'test_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:501\n___________________ ERROR at setup of test_GitRepo_ssh_fetch ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 547\n  @skip_if_on_windows\n  @skip_ssh\n  @with_testrepos('.*basic.*', flavors=['local'])\n  @with_tempfile\n  def test_GitRepo_ssh_fetch(remote_path, repo_path):\nE       fixture 'remote_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:547\n___________________ ERROR at setup of test_GitRepo_ssh_pull ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 579\n  @skip_if_on_windows\n  @skip_ssh\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_ssh_pull(remote_path, repo_path):\nE       fixture 'remote_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:579\n___________________ ERROR at setup of test_GitRepo_ssh_push ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 618\n  @skip_if_on_windows\n  @skip_ssh\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_ssh_push(repo_path, remote_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:618\n________________ ERROR at setup of test_GitRepo_push_n_checkout ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 679\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_push_n_checkout(orig_path, clone_path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:679\n_________________ ERROR at setup of test_GitRepo_remote_update _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 697\n  @with_tempfile\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_remote_update(path1, path2, path3):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:697\n___________________ ERROR at setup of test_GitRepo_get_files ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 743\n  @with_testrepos('.*git.*', flavors=['clone'])\n  @with_tempfile\n  def test_GitRepo_get_files(url, path):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:743\n__________________ ERROR at setup of test_GitRepo_get_toppath __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 791\n  @known_failure_windows\n  @with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n  @with_tempfile(mkdir=True)\n  @with_tempfile\n  def test_GitRepo_get_toppath(repo, tempdir, repo2):\nE       fixture 'repo' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:791\n________________ ERROR at setup of test_GitRepo_get_merge_base _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 869\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_get_merge_base(src):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:869\n____________ ERROR at setup of test_GitRepo_git_get_branch_commits_ ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 905\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_git_get_branch_commits_(src):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:905\n__________________ ERROR at setup of test_get_tracking_branch __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 920\n  @with_testrepos(flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_get_tracking_branch(o_path, c_path):\nE       fixture 'o_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:920\n_________________ ERROR at setup of test_GitRepo_add_submodule _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 979\n  @with_testrepos(\".*basic_git.*\", flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_add_submodule(source, path):\nE       fixture 'source' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:979\n_________________ ERROR at setup of test_GitRepo_count_objects _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1109\n  @with_tempfile\n  def test_GitRepo_count_objects(repo_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1109\n________________ ERROR at setup of test_GitRepo_gitpy_injection ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1161\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_gitpy_injection(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1161\n___________________ ERROR at setup of test_GitRepo_flyweight ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1184\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_flyweight(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1184\n__________ ERROR at setup of test_GitRepo_flyweight_monitoring_inode ___________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1206\n  @with_tempfile(mkdir=True)\n  @with_tempfile()\n  def test_GitRepo_flyweight_monitoring_inode(path, store):\nE       fixture 'store' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1206\n=================================== FAILURES ===================================\n__________________________ test_GitRepo_invalid_path ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexbmeojg02'}\nfilename = '/tmp/datalad_temp_test_GitRepo_invalid_pathmqsh0ke8'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_invalid_path() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________ test_GitRepo_instance_from_existing ______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexfxfumyx4'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'datalad.support.exceptions.CommandError'>, CommandError(\"Failed to run ['git-annex', 'add', '--json', '--json...', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_6nn07ddu'. Exit code=1.\"), <traceback object at 0x721729c90140>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\ndatalad/tests/utils.py:1086: in newfunc\n    testrepos_uris = _get_testrepos_uris(regex, flavors_)\ndatalad/tests/utils.py:1020: in _get_testrepos_uris\n    _basic_annex_test_repo.create()\ndatalad/tests/utils_testrepos.py:90: in create\n    self.populate()\ndatalad/tests/utils_testrepos.py:104: in populate\n    self.create_info_file()\ndatalad/tests/utils_testrepos.py:123: in create_info_file\n    self.create_file('INFO.txt',\ndatalad/tests/utils_testrepos.py:83: in create_file\n    self.repo.add(name, git=True)\ndatalad/support/gitrepo.py:352: in newfunc\n    result = func(self, files_new, *args, **kwargs)\ndatalad/support/annexrepo.py:1345: in add\n    return list(self.add_(\ndatalad/support/annexrepo.py:1433: in add_\n    for r in self._run_annex_command_json(\ndatalad/support/annexrepo.py:2253: in _run_annex_command_json\n    raise e\ndatalad/support/annexrepo.py:2160: in _run_annex_command_json\n    out, err = self._run_annex_command(\ndatalad/support/annexrepo.py:1000: in _run_annex_command\n    return run_gitcommand_on_file_list_chunks(\ndatalad/cmd.py:136: in run_gitcommand_on_file_list_chunks\n    out_, err_ = func(cmd + ['--'] + file_chunk, *args, **kwargs)\ndatalad/cmd.py:1020: in run\n    out, err = super(GitRunner, self).run(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.cmd.GitRunner object at 0x721729ac5400>\ncmd = ['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', ...]\nlog_stdout = <datalad.support.annexrepo.ProcessAnnexProgressIndicators object at 0x72172ae77af0>\nlog_stderr = 'offline', log_online = True, expect_stderr = True\nexpect_fail = True, cwd = None\nenv = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nshell = False, stdin = None\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n    \n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n    \n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies split_cmdline() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n    \n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n    \n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n    \n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n    \n        log_online: bool, optional\n            Whether to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n    \n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n    \n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n    \n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n    \n        env : string, optional\n            Custom environment to pass\n    \n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n    \n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n    \n        Returns\n        -------\n        (stdout, stderr) - bytes!\n    \n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n    \n        popen_env = env or self.env\n        popen_cwd = cwd or self.cwd\n    \n        if popen_cwd and popen_env and 'PWD' in popen_env:\n            # we must have inherited PWD, but cwd was provided, so we must\n            # adjust it\n            popen_env = popen_env.copy()  # to avoid side-effects\n            popen_env['PWD'] = popen_cwd\n    \n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [popen_cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n    \n        if self.protocol.do_execute_ext_commands:\n    \n            if shell is None:\n                shell = isinstance(cmd, str)\n    \n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    split_cmdline(cmd)\n                    if isinstance(cmd, str)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=popen_cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n    \n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n    \n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n    \n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n    \n                # Decoding was delayed to this point\n                def decode_if_not_None(x):\n                    return \"\" if x is None else bytes.decode(x)\n                out = tuple(map(decode_if_not_None, out))\n    \n                status = proc.poll()\n    \n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n    \n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d.%s%s\" \\\n                        % (cmd, \" under %r\" % (popen_cwd), status,\n                           \"\" if log_online else \" out=%s\" % out[0],\n                           \"\" if log_online else \" err=%s\" % out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n>                   raise CommandError(str(cmd), msg, status, out[0], out[1])\nE                   datalad.support.exceptions.CommandError: CommandError: command '['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt']' failed with exitcode 1\nE                   Failed to run ['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_6nn07ddu'. Exit code=1.\nE                   stdout=\nE                   stderr=Invalid option `--include-dotfiles'\nE                   \nE                   Usage: git-annex COMMAND\nE                     git-annex - manage files with git, without checking their contents in\nE                     \nE                     Commonly used commands:\nE                     \nE                     add             PATH ...                  add files to annex\nE                     addurl          URL ...                   add urls to annex\nE                     assistant                                 automatically sync changes\nE                     copy            PATH ...                  copy content of files to/from another repository\nE                     drop            PATH ...                  remove content of files from repository\nE                     edit            PATH ...                  same as unlock\nE                     export          TREEISH                   export a tree of files to a special remote\nE                     get             PATH ...                  make content of annexed files available\nE                     help            COMMAND                   display help\nE                     import          PATH ...|BRANCH[:SUBDIR]  add a tree of files to the repository\nE                     importfeed      URL ...                   import files from podcast feeds\nE                     lock            PATH ...                  undo unlock command\nE                     mirror          PATH ...                  mirror content of files to/from another repository\nE                     move            PATH ...                  move content of files to/from another repository\nE                     multicast                                 multicast file distribution\nE                     remotedaemon                              persistent communication with remotes\nE                     rmurl           FILE URL ...              record file is not available at url\nE                     status          PATH ...                  show the working tree status\nE                     sync            REMOTE ...                synchronize local repository with remotes\nE                     undo            PATH ...                  undo last change to a file or directory\nE                     unlock          PATH ...                  unlock files for modification\nE                     watch                                     watch for changes and autocommit\nE                     webapp                                    launch webapp\nE                     \nE                     Repository setup commands:\nE                     \nE                     adjust                                    enter adjusted branch\nE                     config                                    configuration stored in git-annex branch\nE                     dead            REMOTE ...                hide a lost repository or key\nE                     describe        REMOTE DESC               change description of a repository\nE                     direct                                    switch repository to direct mode (deprecated)\nE                     enable-tor      uid                       enable tor hidden service\nE                     enableremote    NAME PARAM=VALUE ...      enables git-annex to use a remote\nE                     group           REMOTE DESC               add a repository to a group\nE                     groupwanted     GROUP EXPR                get or set groupwanted expression\nE                     indirect                                  switch repository to indirect mode (deprecated)\nE                     init            DESC                      initialize git-annex\nE                     initremote      NAME PARAM=VALUE ...      creates a special (non-git) remote\nE                     mincopies       NUMBER                    configure minimum number of copies\nE                     numcopies       NUMBER                    configure desired number of copies\nE                     p2p                                       configure peer-2-peer links between repositories\nE                     renameremote    NAME NAME                 changes name of special remote\nE                     required        REMOTE EXPR               get or set required content expression\nE                     schedule        REMOTE EXPR               get or set scheduled jobs\nE                     semitrust       REMOTE ...                return repository to default trust level\nE                     trust           REMOTE ...                trust a repository\nE                     ungroup         REMOTE DESC               remove a repository from a group\nE                     untrust         REMOTE ...                do not trust a repository\nE                     vicfg                                     edit configuration in git-annex branch\nE                     wanted          REMOTE EXPR               get or set preferred content expression\nE                     \nE                     Repository maintenance commands:\nE                     \nE                     addunused       NUM|RANGE ...             add back unused files\nE                     dropunused      NUM|RANGE ...             drop unused file content\nE                     expire          REMOTE:TIME ...           expire inactive repositories\nE                     fix             PATH ...                  fix up links to annexed content\nE                     forget                                    prune git-annex branch history\nE                     fsck            PATH ...                  find and fix problems\nE                     merge           REF                       merge changes from remotes\nE                     repair                                    recover broken git repository\nE                     unused                                    look for unused file content\nE                     upgrade                                   upgrade repository\nE                     \nE                     Query commands:\nE                     \nE                     find            PATH ...                  lists available files\nE                     info            ITEM ...                  information about an item or the repository\nE                     inprogress      PATH ...                  access files while they're being downloaded\nE                     list            PATH ...                  show which remotes contain files\nE                     log             PATH ...                  shows location log\nE                     map                                       generate map of repositories\nE                     version                                   show version info\nE                     whereis         PATH ...                  lists repositories that have file content\nE                     \nE                     Metadata commands:\nE                     \nE                     metadata        PATH ...                  sets or gets metadata of a file\nE                     vadd            FIELD=GLOB ...            add subdirs to current view\nE                     vcycle                                    switch view to next layout\nE                     vfilter         FIELD=VALUE ...           filter current view\nE                     view            FIELD=VALUE ...           enter a view branch\nE                     vpop            NUMBER                    switch back to previous view\nE                     \nE                     Utility commands:\nE                     \nE                     migrate         PATH ...                  switch data to different backend\nE                     reinit          UUID|DESC                 initialize repository, reusing old UUID\nE                     reinject        SRC DEST ...              inject content of file back into annex\nE                     unannex         PATH ...                  undo accidental add command\nE                     uninit          PATH ...                  de-initialize git-annex and clean out repository\nE                     \nE                     Plumbing commands:\nE                     \nE                     calckey         FILE ...                  calulate key for a file\nE                     checkpresentkey KEY REMOTE                check if key is present in remote\nE                     contentlocation KEY ...                   looks up content for a key\nE                     diffdriver      -- cmd --                 external git diff driver shim\nE                     dropkey         KEY ...                   drops annexed content for specified keys\nE                     examinekey      KEY ...                   prints information from a key\nE                     findref         REF                       lists files in a git ref (deprecated)\nE                     fromkey         KEY PATH ...              adds a file using a specific key\nE                     lookupkey       FILE ...                  looks up key used for file\nE                     matchexpression EXPR                      checks if a preferred content expression matches\nE                     post-receive                              run by git post-receive hook\nE                     pre-commit      PATH ...                  run by git pre-commit hook\nE                     proxy           -- git command            safely bypass direct mode guard (deprecated)\nE                     readpresentkey  KEY UUID                  read records of where key is present\nE                     registerurl     KEY URL                   registers an url for a key\nE                     rekey           PATH KEY ...              change keys used for files\nE                     resolvemerge                              resolve merge conflicts\nE                     setkey          KEY PATH                  sets annexed content for a key\nE                     setpresentkey   KEY UUID [1|0]            change records of where key is present\nE                     smudge          FILE                      git smudge filter\nE                     transferkey     KEY                       transfers a key from or to a remote\nE                     transferkeys                              transfers keys (deprecated)\nE                     transferrer                               transfers content\nE                     \nE                     Testing commands:\nE                     \nE                     benchmark                                 benchmark git-annex commands\nE                     fuzztest                                  generates fuzz test files\nE                     test                                      run built-in test suite\nE                     testremote      REMOTE                    test transfers to/from a remote\n\ndatalad/cmd.py:854: CommandError\n__________________________ test_GitRepo_init_options ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvb6cn2hw'}\nfilename = '/tmp/datalad_temp_test_GitRepo_init_optionsxsll2bnv'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_init_options() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________ test_init_fail_under_known_subdir _______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annex23593p2h'}\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        if cond:\n            if method == 'raise':\n                raise SkipTest(msg if msg else \"condition was True\")\n            elif method == 'pass':\n                print(msg if msg else \"condition was True\")\n                return\n>       return func(*args, **kwargs)\n\ndatalad/tests/utils.py:222: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annex23593p2h'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_init_fail_under_known_subdir'}\nd = '/tmp/datalad_temp_tree_test_init_fail_under_known_subdirexrw5qpy'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_init_fail_under_known_subdir() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_____________________________ test_GitRepo_remove ______________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexdpdhaz0_'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'TypeError'>, TypeError(\"test_GitRepo_remove() got multiple values for argument 'path'\"), <traceback object at 0x72172972f980>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexdpdhaz0_'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_GitRepo_remove'}\nd = '/tmp/datalad_temp_tree_test_GitRepo_remove5pdpf9df'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_GitRepo_remove() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_____________________________ test_GitRepo_commit ______________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexbw3_3ibp'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'TypeError'>, TypeError(\"test_GitRepo_commit() got multiple values for argument 'path'\"), <traceback object at 0x721729b34380>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexbw3_3ibp'}\nfilename = '/tmp/datalad_temp_test_GitRepo_commitmp8bggug'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________________ test_GitRepo_dirty ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex47p6wjnx'}\nfilename = '/tmp/datalad_temp_test_GitRepo_dirty949rbg66'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________________ test_submodule_deinit _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvj0ls9i_'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_eb8y0vkz'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_eb8y0vkz'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_submodule_deinit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:1104: TypeError\n_________________ test_get_submodules_parent_on_unborn_branch __________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexcgdh4x_f'}\nfilename = '/tmp/datalad_temp_test_get_submodules_parent_on_unborn_branchw3c7wtxo'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_submodules_parent_on_unborn_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________________ test_optimized_cloning ____________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexz6v7s39d'}\n\n    @wraps(func)\n    @attr('skip_if_on_windows')\n    def newfunc(*args, **kwargs):\n        check_and_raise()\n>       return func(*args, **kwargs)\n\ndatalad/tests/utils.py:188: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexz6v7s39d'}\nfilename = '/tmp/datalad_temp_test_optimized_cloningkyej5n4d'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_optimized_cloning() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________________ test_GitRepo_gitignore ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex48zdua2h'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_GitRepo_gitignore'}\nd = '/tmp/datalad_temp_tree_test_GitRepo_gitignorejwcggzon'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_GitRepo_gitignore() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_________________________ test_GitRepo_set_remote_url __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexfmbhqcee'}\nfilename = '/tmp/datalad_temp_test_GitRepo_set_remote_urlfd_o1bjc'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_set_remote_url() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________________ test_gitattributes ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex6jgf4x6o'}\nfilename = '/tmp/datalad_temp_test_gitattributeswkt25esd'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_gitattributes() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________________ test_get_hexsha_tag ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex46chetqw'}\nfilename = '/tmp/datalad_temp_test_get_hexsha_tagjawwz07o'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_hexsha_tag() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n________________________________ test_get_tags _________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexgicevnug'}\nfilename = '/tmp/datalad_temp_test_get_tagsl6sx4fe5'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_tags() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________________ test_get_commit_date _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexmtv_9lq0'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_get_commit_date'}\nd = '/tmp/datalad_temp_tree_test_get_commit_datesh69nlse'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_get_commit_date() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_______________________________ test_fake_dates ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexmk_fab4g'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_fake_dates'}\nd = '/tmp/datalad_temp_tree_test_fake_datesx9szbzyb'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_fake_dates() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_________________________ test_custom_runner_protocol __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexfaru2m90'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_custom_runner_protocol'}\nd = '/tmp/datalad_temp_tree_test_custom_runner_protocolic3l9obs'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_custom_runner_protocol() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n________________________________ test_duecredit ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexnmaq_yg1'}\nfilename = '/tmp/datalad_temp_test_duecreditx177oy6c'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_duecredit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n__________________________ test_GitRepo_get_revisions __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex8klnlemw'}\nfilename = '/tmp/datalad_temp_test_GitRepo_get_revisions6fqddbaz'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_get_revisions() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________ test_gitrepo_add_to_git_with_annex_v7 _____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexhp2dap75'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_gitrepo_add_to_git_with_annex_v7'}\nd = '/tmp/datalad_temp_tree_test_gitrepo_add_to_git_with_annex_v7gmqq9oqe'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_gitrepo_add_to_git_with_annex_v7() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n________________________ test_gitrepo_call_git_methods _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex_vhnydip'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_gitrepo_call_git_methods'}\nd = '/tmp/datalad_temp_tree_test_gitrepo_call_git_methodsta9ggjoy'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_gitrepo_call_git_methods() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_GitRepo_files_decorator\nPASSED r2e_tests/test_1.py::test_to_options\nPASSED r2e_tests/test_1.py::test_to_options_from_gitpython\nPASSED r2e_tests/test_1.py::test_guard_BadName\nSKIPPED [2] .venv/lib/python3.9/site-packages/_pytest/unittest.py:385: TODO\nXFAIL r2e_tests/test_1.py::test_GitRepo_update_submodule_init_adjust_branch - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_GitRepo_update_submodule_init_adjust_branch will be ignored\nXFAIL r2e_tests/test_1.py::test_protocols - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_protocols will be ignored\nERROR r2e_tests/test_1.py::test_GitRepo_instance_from_clone\nERROR r2e_tests/test_1.py::test_GitRepo_instance_from_not_existing\nERROR r2e_tests/test_1.py::test_GitRepo_equals\nERROR r2e_tests/test_1.py::test_GitRepo_add\nERROR r2e_tests/test_1.py::test_GitRepo_get_indexed_files\nERROR r2e_tests/test_1.py::test_normalize_path\nERROR r2e_tests/test_1.py::test_GitRepo_remote_add\nERROR r2e_tests/test_1.py::test_GitRepo_remote_remove\nERROR r2e_tests/test_1.py::test_GitRepo_get_remote_url\nERROR r2e_tests/test_1.py::test_GitRepo_pull\nERROR r2e_tests/test_1.py::test_GitRepo_fetch\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_fetch\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_pull\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_push\nERROR r2e_tests/test_1.py::test_GitRepo_push_n_checkout\nERROR r2e_tests/test_1.py::test_GitRepo_remote_update\nERROR r2e_tests/test_1.py::test_GitRepo_get_files\nERROR r2e_tests/test_1.py::test_GitRepo_get_toppath\nERROR r2e_tests/test_1.py::test_GitRepo_get_merge_base\nERROR r2e_tests/test_1.py::test_GitRepo_git_get_branch_commits_\nERROR r2e_tests/test_1.py::test_get_tracking_branch\nERROR r2e_tests/test_1.py::test_GitRepo_add_submodule\nERROR r2e_tests/test_1.py::test_GitRepo_count_objects\nERROR r2e_tests/test_1.py::test_GitRepo_gitpy_injection\nERROR r2e_tests/test_1.py::test_GitRepo_flyweight\nERROR r2e_tests/test_1.py::test_GitRepo_flyweight_monitoring_inode\nFAILED r2e_tests/test_1.py::test_GitRepo_invalid_path - TypeError: test_GitRe...\nFAILED r2e_tests/test_1.py::test_GitRepo_instance_from_existing - datalad.sup...\nFAILED r2e_tests/test_1.py::test_GitRepo_init_options - TypeError: test_GitRe...\nFAILED r2e_tests/test_1.py::test_init_fail_under_known_subdir - TypeError: te...\nFAILED r2e_tests/test_1.py::test_GitRepo_remove - TypeError: test_GitRepo_rem...\nFAILED r2e_tests/test_1.py::test_GitRepo_commit - TypeError: test_GitRepo_com...\nFAILED r2e_tests/test_1.py::test_GitRepo_dirty - TypeError: test_GitRepo_dirt...\nFAILED r2e_tests/test_1.py::test_submodule_deinit - TypeError: test_submodule...\nFAILED r2e_tests/test_1.py::test_get_submodules_parent_on_unborn_branch - Typ...\nFAILED r2e_tests/test_1.py::test_optimized_cloning - TypeError: test_optimize...\nFAILED r2e_tests/test_1.py::test_GitRepo_gitignore - TypeError: test_GitRepo_...\nFAILED r2e_tests/test_1.py::test_GitRepo_set_remote_url - TypeError: test_Git...\nFAILED r2e_tests/test_1.py::test_gitattributes - TypeError: test_gitattribute...\nFAILED r2e_tests/test_1.py::test_get_hexsha_tag - TypeError: test_get_hexsha_...\nFAILED r2e_tests/test_1.py::test_get_tags - TypeError: test_get_tags() got mu...\nFAILED r2e_tests/test_1.py::test_get_commit_date - TypeError: test_get_commit...\nFAILED r2e_tests/test_1.py::test_fake_dates - TypeError: test_fake_dates() go...\nFAILED r2e_tests/test_1.py::test_custom_runner_protocol - TypeError: test_cus...\nFAILED r2e_tests/test_1.py::test_duecredit - TypeError: test_duecredit() got ...\nFAILED r2e_tests/test_1.py::test_GitRepo_get_revisions - TypeError: test_GitR...\nFAILED r2e_tests/test_1.py::test_gitrepo_add_to_git_with_annex_v7 - TypeError...\nFAILED r2e_tests/test_1.py::test_gitrepo_call_git_methods - TypeError: test_g...\n======== 22 failed, 4 passed, 2 skipped, 2 xfailed, 26 errors in 4.67s =========\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.9.21, pytest-8.3.4, pluggy-1.5.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133\nplugins: cov-6.0.0\ncollected 56 items\n\nr2e_tests/test_1.py FEFEFFEEFFEE.EEEEEEEEEEEEFEEEFEsxsFF.EFEEEFFFFFFF.FF [ 92%]\nFFFx                                                                     [100%]\n\n==================================== ERRORS ====================================\n______________ ERROR at setup of test_GitRepo_instance_from_clone ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 97\n  @assert_cwd_unchanged\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_instance_from_clone(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:97\n__________ ERROR at setup of test_GitRepo_instance_from_not_existing ___________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 126\n  @assert_cwd_unchanged\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_instance_from_not_existing(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:126\n____________________ ERROR at setup of test_GitRepo_equals _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 186\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_equals(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:186\n______________________ ERROR at setup of test_GitRepo_add ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 200\n  @known_failure_windows\n  @assert_cwd_unchanged\n  @with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_add(src, path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:200\n_______________ ERROR at setup of test_GitRepo_get_indexed_files _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 294\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_get_indexed_files(src, path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:294\n____________________ ERROR at setup of test_normalize_path _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 311\n  @with_tree([\n      ('empty', ''),\n      ('d1', (\n          ('empty', ''),\n          ('d2',\n              (('empty', ''),\n               )),\n          )),\n      ])\n  @assert_cwd_unchanged(ok_to_chdir=True)\n  def test_normalize_path(git_path):\nE       fixture 'git_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:311\n__________________ ERROR at setup of test_GitRepo_remote_add ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 428\n  @skip_if_no_network\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_remote_add(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:428\n_________________ ERROR at setup of test_GitRepo_remote_remove _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 445\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_remote_remove(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:445\n________________ ERROR at setup of test_GitRepo_get_remote_url _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 458\n  @known_failure_windows\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_GitRepo_get_remote_url(orig_path, path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:458\n_____________________ ERROR at setup of test_GitRepo_pull ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 470\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_pull(test_path, orig_path, clone_path):\nE       fixture 'test_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:470\n_____________________ ERROR at setup of test_GitRepo_fetch _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 501\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_fetch(test_path, orig_path, clone_path):\nE       fixture 'test_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:501\n___________________ ERROR at setup of test_GitRepo_ssh_fetch ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 547\n  @skip_if_on_windows\n  @skip_ssh\n  @with_testrepos('.*basic.*', flavors=['local'])\n  @with_tempfile\n  def test_GitRepo_ssh_fetch(remote_path, repo_path):\nE       fixture 'remote_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:547\n___________________ ERROR at setup of test_GitRepo_ssh_pull ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 579\n  @skip_if_on_windows\n  @skip_ssh\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_ssh_pull(remote_path, repo_path):\nE       fixture 'remote_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:579\n___________________ ERROR at setup of test_GitRepo_ssh_push ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 618\n  @skip_if_on_windows\n  @skip_ssh\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_ssh_push(repo_path, remote_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:618\n________________ ERROR at setup of test_GitRepo_push_n_checkout ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 679\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_push_n_checkout(orig_path, clone_path):\nE       fixture 'orig_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:679\n_________________ ERROR at setup of test_GitRepo_remote_update _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 697\n  @with_tempfile\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_remote_update(path1, path2, path3):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:697\n___________________ ERROR at setup of test_GitRepo_get_files ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 743\n  @with_testrepos('.*git.*', flavors=['clone'])\n  @with_tempfile\n  def test_GitRepo_get_files(url, path):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:743\n__________________ ERROR at setup of test_GitRepo_get_toppath __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 791\n  @known_failure_windows\n  @with_testrepos('.*git.*', flavors=local_testrepo_flavors)\n  @with_tempfile(mkdir=True)\n  @with_tempfile\n  def test_GitRepo_get_toppath(repo, tempdir, repo2):\nE       fixture 'repo' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:791\n________________ ERROR at setup of test_GitRepo_get_merge_base _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 869\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_get_merge_base(src):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:869\n____________ ERROR at setup of test_GitRepo_git_get_branch_commits_ ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 905\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_git_get_branch_commits_(src):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:905\n__________________ ERROR at setup of test_get_tracking_branch __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 920\n  @with_testrepos(flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_get_tracking_branch(o_path, c_path):\nE       fixture 'o_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:920\n_________________ ERROR at setup of test_GitRepo_add_submodule _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 979\n  @with_testrepos(\".*basic_git.*\", flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_add_submodule(source, path):\nE       fixture 'source' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:979\n_________________ ERROR at setup of test_GitRepo_count_objects _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1109\n  @with_tempfile\n  def test_GitRepo_count_objects(repo_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1109\n________________ ERROR at setup of test_GitRepo_gitpy_injection ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1161\n  @with_tempfile\n  @with_tempfile\n  def test_GitRepo_gitpy_injection(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1161\n___________________ ERROR at setup of test_GitRepo_flyweight ___________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1184\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_GitRepo_flyweight(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1184\n__________ ERROR at setup of test_GitRepo_flyweight_monitoring_inode ___________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py, line 1206\n  @with_tempfile(mkdir=True)\n  @with_tempfile()\n  def test_GitRepo_flyweight_monitoring_inode(path, store):\nE       fixture 'store' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133/r2e_tests/test_1.py:1206\n=================================== FAILURES ===================================\n__________________________ test_GitRepo_invalid_path ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexj1tque3u'}\nfilename = '/tmp/datalad_temp_test_GitRepo_invalid_pathoodprkb8'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_invalid_path() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________ test_GitRepo_instance_from_existing ______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexcn3xlrvz'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'datalad.support.exceptions.CommandError'>, CommandError(\"Failed to run ['git-annex', 'add', '--json', '--json...', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_4mh6uddi'. Exit code=1.\"), <traceback object at 0x710feca48800>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\ndatalad/tests/utils.py:1086: in newfunc\n    testrepos_uris = _get_testrepos_uris(regex, flavors_)\ndatalad/tests/utils.py:1020: in _get_testrepos_uris\n    _basic_annex_test_repo.create()\ndatalad/tests/utils_testrepos.py:90: in create\n    self.populate()\ndatalad/tests/utils_testrepos.py:104: in populate\n    self.create_info_file()\ndatalad/tests/utils_testrepos.py:123: in create_info_file\n    self.create_file('INFO.txt',\ndatalad/tests/utils_testrepos.py:83: in create_file\n    self.repo.add(name, git=True)\ndatalad/support/gitrepo.py:320: in newfunc\n    result = func(self, files_new, *args, **kwargs)\ndatalad/support/annexrepo.py:1345: in add\n    return list(self.add_(\ndatalad/support/annexrepo.py:1433: in add_\n    for r in self._run_annex_command_json(\ndatalad/support/annexrepo.py:2253: in _run_annex_command_json\n    raise e\ndatalad/support/annexrepo.py:2160: in _run_annex_command_json\n    out, err = self._run_annex_command(\ndatalad/support/annexrepo.py:1000: in _run_annex_command\n    return run_gitcommand_on_file_list_chunks(\ndatalad/cmd.py:136: in run_gitcommand_on_file_list_chunks\n    out_, err_ = func(cmd + ['--'] + file_chunk, *args, **kwargs)\ndatalad/cmd.py:1020: in run\n    out, err = super(GitRunner, self).run(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.cmd.GitRunner object at 0x710fec9ec580>\ncmd = ['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', ...]\nlog_stdout = <datalad.support.annexrepo.ProcessAnnexProgressIndicators object at 0x710fed990e80>\nlog_stderr = 'offline', log_online = True, expect_stderr = True\nexpect_fail = True, cwd = None\nenv = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nshell = False, stdin = None\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n    \n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n    \n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies split_cmdline() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n    \n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n    \n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n    \n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n    \n        log_online: bool, optional\n            Whether to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n    \n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n    \n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n    \n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n    \n        env : string, optional\n            Custom environment to pass\n    \n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n    \n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n    \n        Returns\n        -------\n        (stdout, stderr) - bytes!\n    \n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n    \n        popen_env = env or self.env\n        popen_cwd = cwd or self.cwd\n    \n        if popen_cwd and popen_env and 'PWD' in popen_env:\n            # we must have inherited PWD, but cwd was provided, so we must\n            # adjust it\n            popen_env = popen_env.copy()  # to avoid side-effects\n            popen_env['PWD'] = popen_cwd\n    \n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [popen_cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n    \n        if self.protocol.do_execute_ext_commands:\n    \n            if shell is None:\n                shell = isinstance(cmd, str)\n    \n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    split_cmdline(cmd)\n                    if isinstance(cmd, str)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=popen_cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n    \n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n    \n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n    \n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n    \n                # Decoding was delayed to this point\n                def decode_if_not_None(x):\n                    return \"\" if x is None else bytes.decode(x)\n                out = tuple(map(decode_if_not_None, out))\n    \n                status = proc.poll()\n    \n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n    \n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d.%s%s\" \\\n                        % (cmd, \" under %r\" % (popen_cwd), status,\n                           \"\" if log_online else \" out=%s\" % out[0],\n                           \"\" if log_online else \" err=%s\" % out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n>                   raise CommandError(str(cmd), msg, status, out[0], out[1])\nE                   datalad.support.exceptions.CommandError: CommandError: command '['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt']' failed with exitcode 1\nE                   Failed to run ['git-annex', 'add', '--json', '--json-error-messages', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_4mh6uddi'. Exit code=1.\nE                   stdout=\nE                   stderr=Invalid option `--include-dotfiles'\nE                   \nE                   Usage: git-annex COMMAND\nE                     git-annex - manage files with git, without checking their contents in\nE                     \nE                     Commonly used commands:\nE                     \nE                     add             PATH ...                  add files to annex\nE                     addurl          URL ...                   add urls to annex\nE                     assistant                                 automatically sync changes\nE                     copy            PATH ...                  copy content of files to/from another repository\nE                     drop            PATH ...                  remove content of files from repository\nE                     edit            PATH ...                  same as unlock\nE                     export          TREEISH                   export a tree of files to a special remote\nE                     get             PATH ...                  make content of annexed files available\nE                     help            COMMAND                   display help\nE                     import          PATH ...|BRANCH[:SUBDIR]  add a tree of files to the repository\nE                     importfeed      URL ...                   import files from podcast feeds\nE                     lock            PATH ...                  undo unlock command\nE                     mirror          PATH ...                  mirror content of files to/from another repository\nE                     move            PATH ...                  move content of files to/from another repository\nE                     multicast                                 multicast file distribution\nE                     remotedaemon                              persistent communication with remotes\nE                     rmurl           FILE URL ...              record file is not available at url\nE                     status          PATH ...                  show the working tree status\nE                     sync            REMOTE ...                synchronize local repository with remotes\nE                     undo            PATH ...                  undo last change to a file or directory\nE                     unlock          PATH ...                  unlock files for modification\nE                     watch                                     watch for changes and autocommit\nE                     webapp                                    launch webapp\nE                     \nE                     Repository setup commands:\nE                     \nE                     adjust                                    enter adjusted branch\nE                     config                                    configuration stored in git-annex branch\nE                     dead            REMOTE ...                hide a lost repository or key\nE                     describe        REMOTE DESC               change description of a repository\nE                     direct                                    switch repository to direct mode (deprecated)\nE                     enable-tor      uid                       enable tor hidden service\nE                     enableremote    NAME PARAM=VALUE ...      enables git-annex to use a remote\nE                     group           REMOTE DESC               add a repository to a group\nE                     groupwanted     GROUP EXPR                get or set groupwanted expression\nE                     indirect                                  switch repository to indirect mode (deprecated)\nE                     init            DESC                      initialize git-annex\nE                     initremote      NAME PARAM=VALUE ...      creates a special (non-git) remote\nE                     mincopies       NUMBER                    configure minimum number of copies\nE                     numcopies       NUMBER                    configure desired number of copies\nE                     p2p                                       configure peer-2-peer links between repositories\nE                     renameremote    NAME NAME                 changes name of special remote\nE                     required        REMOTE EXPR               get or set required content expression\nE                     schedule        REMOTE EXPR               get or set scheduled jobs\nE                     semitrust       REMOTE ...                return repository to default trust level\nE                     trust           REMOTE ...                trust a repository\nE                     ungroup         REMOTE DESC               remove a repository from a group\nE                     untrust         REMOTE ...                do not trust a repository\nE                     vicfg                                     edit configuration in git-annex branch\nE                     wanted          REMOTE EXPR               get or set preferred content expression\nE                     \nE                     Repository maintenance commands:\nE                     \nE                     addunused       NUM|RANGE ...             add back unused files\nE                     dropunused      NUM|RANGE ...             drop unused file content\nE                     expire          REMOTE:TIME ...           expire inactive repositories\nE                     fix             PATH ...                  fix up links to annexed content\nE                     forget                                    prune git-annex branch history\nE                     fsck            PATH ...                  find and fix problems\nE                     merge           REF                       merge changes from remotes\nE                     repair                                    recover broken git repository\nE                     unused                                    look for unused file content\nE                     upgrade                                   upgrade repository\nE                     \nE                     Query commands:\nE                     \nE                     find            PATH ...                  lists available files\nE                     info            ITEM ...                  information about an item or the repository\nE                     inprogress      PATH ...                  access files while they're being downloaded\nE                     list            PATH ...                  show which remotes contain files\nE                     log             PATH ...                  shows location log\nE                     map                                       generate map of repositories\nE                     version                                   show version info\nE                     whereis         PATH ...                  lists repositories that have file content\nE                     \nE                     Metadata commands:\nE                     \nE                     metadata        PATH ...                  sets or gets metadata of a file\nE                     vadd            FIELD=GLOB ...            add subdirs to current view\nE                     vcycle                                    switch view to next layout\nE                     vfilter         FIELD=VALUE ...           filter current view\nE                     view            FIELD=VALUE ...           enter a view branch\nE                     vpop            NUMBER                    switch back to previous view\nE                     \nE                     Utility commands:\nE                     \nE                     migrate         PATH ...                  switch data to different backend\nE                     reinit          UUID|DESC                 initialize repository, reusing old UUID\nE                     reinject        SRC DEST ...              inject content of file back into annex\nE                     unannex         PATH ...                  undo accidental add command\nE                     uninit          PATH ...                  de-initialize git-annex and clean out repository\nE                     \nE                     Plumbing commands:\nE                     \nE                     calckey         FILE ...                  calulate key for a file\nE                     checkpresentkey KEY REMOTE                check if key is present in remote\nE                     contentlocation KEY ...                   looks up content for a key\nE                     diffdriver      -- cmd --                 external git diff driver shim\nE                     dropkey         KEY ...                   drops annexed content for specified keys\nE                     examinekey      KEY ...                   prints information from a key\nE                     findref         REF                       lists files in a git ref (deprecated)\nE                     fromkey         KEY PATH ...              adds a file using a specific key\nE                     lookupkey       FILE ...                  looks up key used for file\nE                     matchexpression EXPR                      checks if a preferred content expression matches\nE                     post-receive                              run by git post-receive hook\nE                     pre-commit      PATH ...                  run by git pre-commit hook\nE                     proxy           -- git command            safely bypass direct mode guard (deprecated)\nE                     readpresentkey  KEY UUID                  read records of where key is present\nE                     registerurl     KEY URL                   registers an url for a key\nE                     rekey           PATH KEY ...              change keys used for files\nE                     resolvemerge                              resolve merge conflicts\nE                     setkey          KEY PATH                  sets annexed content for a key\nE                     setpresentkey   KEY UUID [1|0]            change records of where key is present\nE                     smudge          FILE                      git smudge filter\nE                     transferkey     KEY                       transfers a key from or to a remote\nE                     transferkeys                              transfers keys (deprecated)\nE                     transferrer                               transfers content\nE                     \nE                     Testing commands:\nE                     \nE                     benchmark                                 benchmark git-annex commands\nE                     fuzztest                                  generates fuzz test files\nE                     test                                      run built-in test suite\nE                     testremote      REMOTE                    test transfers to/from a remote\n\ndatalad/cmd.py:854: CommandError\n__________________________ test_GitRepo_init_options ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexzbvdtje9'}\nfilename = '/tmp/datalad_temp_test_GitRepo_init_optionsr759tu__'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_init_options() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________ test_init_fail_under_known_subdir _______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexoboif4ha'}\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        if cond:\n            if method == 'raise':\n                raise SkipTest(msg if msg else \"condition was True\")\n            elif method == 'pass':\n                print(msg if msg else \"condition was True\")\n                return\n>       return func(*args, **kwargs)\n\ndatalad/tests/utils.py:222: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexoboif4ha'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_init_fail_under_known_subdir'}\nd = '/tmp/datalad_temp_tree_test_init_fail_under_known_subdirrktew00j'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_init_fail_under_known_subdir() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_____________________________ test_GitRepo_remove ______________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annex35km7u7v'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'TypeError'>, TypeError(\"test_GitRepo_remove() got multiple values for argument 'path'\"), <traceback object at 0x710fec52bd40>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annex35km7u7v'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_GitRepo_remove'}\nd = '/tmp/datalad_temp_tree_test_GitRepo_removejs1l4o6i'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_GitRepo_remove() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_____________________________ test_GitRepo_commit ______________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexl_yh3j5i'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\nexc_info = (<class 'TypeError'>, TypeError(\"test_GitRepo_commit() got multiple values for argument 'path'\"), <traceback object at 0x710fec92ca80>)\nutils_pwd_mode = 'PWD'\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_8fb58b5da3dcb5ca940664b159653baa9be85133'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        # record previous state of PWD handling\n        utils_pwd_mode = utils._pwd_mode\n        try:\n            ret = func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            utils._pwd_mode = utils_pwd_mode\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            # Above chpwd could also trigger the change of _pwd_mode, so we\n            # would need to reset it again since we know that it is all kosher\n            utils._pwd_mode = utils_pwd_mode\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           raise exc_info[1]\n\ndatalad/tests/utils.py:1227: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1199: in newfunc\n    ret = func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexl_yh3j5i'}\nfilename = '/tmp/datalad_temp_test_GitRepo_commitsznb3fbt'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________________ test_GitRepo_dirty ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2qwp5exz'}\nfilename = '/tmp/datalad_temp_test_GitRepo_dirtyn6gd_2g1'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________________ test_submodule_deinit _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexna3dq7jg'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_cp1q6ys3'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_cp1q6ys3'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_submodule_deinit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:1104: TypeError\n_________________ test_get_submodules_parent_on_unborn_branch __________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex6zffckb2'}\nfilename = '/tmp/datalad_temp_test_get_submodules_parent_on_unborn_branchd70ebrke'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_submodules_parent_on_unborn_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_______________________________ test_to_options ________________________________\n\n    def test_to_options():\n    \n        class Some(object):\n    \n            def cmd_func(self, git_options=None, annex_options=None, options=None):\n    \n                git_options = git_options[:] if git_options else []\n                annex_options = annex_options[:] if annex_options else []\n                options = options[:] if options else []\n    \n                faked_cmd_call = ['git'] + git_options + ['annex'] + \\\n                                 annex_options + ['my_cmd'] + options\n    \n                return faked_cmd_call\n    \n>       eq_(Some().cmd_func(options=to_options(m=\"bla\", force=True)),\n            ['git', 'annex', 'my_cmd', '--force', '-m', 'bla'])\n\nr2e_tests/test_1.py:1069: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = ['git', 'annex', 'my_cmd', '-m', 'bla', '--force']\nb = ['git', 'annex', 'my_cmd', '--force', '-m', 'bla'], msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: ['git', 'annex', 'my_cmd', '-m', 'bla', '--force'] != ['git', 'annex', 'my_cmd', '--force', '-m', 'bla']\n\n.venv/lib/python3.9/site-packages/nose/tools/trivial.py:29: AssertionError\n____________________________ test_optimized_cloning ____________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexad_l4dnv'}\n\n    @wraps(func)\n    @attr('skip_if_on_windows')\n    def newfunc(*args, **kwargs):\n        check_and_raise()\n>       return func(*args, **kwargs)\n\ndatalad/tests/utils.py:188: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexad_l4dnv'}\nfilename = '/tmp/datalad_temp_test_optimized_cloning12ixvx72'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_optimized_cloning() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________________ test_GitRepo_gitignore ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvjn6fld1'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_GitRepo_gitignore'}\nd = '/tmp/datalad_temp_tree_test_GitRepo_gitignorej9kq_cfs'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_GitRepo_gitignore() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_________________________ test_GitRepo_set_remote_url __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexpn4j0kxf'}\nfilename = '/tmp/datalad_temp_test_GitRepo_set_remote_urlmehd4qpv'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_set_remote_url() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n______________________________ test_gitattributes ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexaqvnnrae'}\nfilename = '/tmp/datalad_temp_test_gitattributesyoxgc859'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_gitattributes() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________________ test_get_hexsha_tag ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexj3djeiia'}\nfilename = '/tmp/datalad_temp_test_get_hexsha_tagseahmeu5'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_hexsha_tag() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n________________________________ test_get_tags _________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvfyesavz'}\nfilename = '/tmp/datalad_temp_test_get_tagsh8tmn2uy'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_get_tags() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n_____________________________ test_get_commit_date _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2rk5uaxa'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_get_commit_date'}\nd = '/tmp/datalad_temp_tree_test_get_commit_dates6u3mr7a'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_get_commit_date() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_______________________________ test_fake_dates ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2v2hewo9'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_fake_dates'}\nd = '/tmp/datalad_temp_tree_test_fake_datessls5qnka'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_fake_dates() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n_________________________ test_custom_runner_protocol __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexp_4s7ixe'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_custom_runner_protocol'}\nd = '/tmp/datalad_temp_tree_test_custom_runner_protocoley4oobwh'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_custom_runner_protocol() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n________________________________ test_duecredit ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexl4x8rn6r'}\nfilename = '/tmp/datalad_temp_test_duecreditgd4dnatx'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_duecredit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n__________________________ test_GitRepo_get_revisions __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex7zmbfwy9'}\nfilename = '/tmp/datalad_temp_test_GitRepo_get_revisionsgl60ch0j'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_GitRepo_get_revisions() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:759: TypeError\n____________________ test_gitrepo_add_to_git_with_annex_v7 _____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex1w9nasxf'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_gitrepo_add_to_git_with_annex_v7'}\nd = '/tmp/datalad_temp_tree_test_gitrepo_add_to_git_with_annex_v7oqxm6f3j'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_gitrepo_add_to_git_with_annex_v7() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n________________________ test_gitrepo_call_git_methods _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexkxf_ca7y'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_gitrepo_call_git_methods'}\nd = '/tmp/datalad_temp_tree_test_gitrepo_call_git_methodstsycv5ne'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_gitrepo_call_git_methods() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:586: TypeError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_GitRepo_files_decorator\nPASSED r2e_tests/test_1.py::test_to_options_from_gitpython\nPASSED r2e_tests/test_1.py::test_guard_BadName\nSKIPPED [2] .venv/lib/python3.9/site-packages/_pytest/unittest.py:385: TODO\nXFAIL r2e_tests/test_1.py::test_GitRepo_update_submodule_init_adjust_branch - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_GitRepo_update_submodule_init_adjust_branch will be ignored\nXFAIL r2e_tests/test_1.py::test_protocols - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_protocols will be ignored\nERROR r2e_tests/test_1.py::test_GitRepo_instance_from_clone\nERROR r2e_tests/test_1.py::test_GitRepo_instance_from_not_existing\nERROR r2e_tests/test_1.py::test_GitRepo_equals\nERROR r2e_tests/test_1.py::test_GitRepo_add\nERROR r2e_tests/test_1.py::test_GitRepo_get_indexed_files\nERROR r2e_tests/test_1.py::test_normalize_path\nERROR r2e_tests/test_1.py::test_GitRepo_remote_add\nERROR r2e_tests/test_1.py::test_GitRepo_remote_remove\nERROR r2e_tests/test_1.py::test_GitRepo_get_remote_url\nERROR r2e_tests/test_1.py::test_GitRepo_pull\nERROR r2e_tests/test_1.py::test_GitRepo_fetch\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_fetch\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_pull\nERROR r2e_tests/test_1.py::test_GitRepo_ssh_push\nERROR r2e_tests/test_1.py::test_GitRepo_push_n_checkout\nERROR r2e_tests/test_1.py::test_GitRepo_remote_update\nERROR r2e_tests/test_1.py::test_GitRepo_get_files\nERROR r2e_tests/test_1.py::test_GitRepo_get_toppath\nERROR r2e_tests/test_1.py::test_GitRepo_get_merge_base\nERROR r2e_tests/test_1.py::test_GitRepo_git_get_branch_commits_\nERROR r2e_tests/test_1.py::test_get_tracking_branch\nERROR r2e_tests/test_1.py::test_GitRepo_add_submodule\nERROR r2e_tests/test_1.py::test_GitRepo_count_objects\nERROR r2e_tests/test_1.py::test_GitRepo_gitpy_injection\nERROR r2e_tests/test_1.py::test_GitRepo_flyweight\nERROR r2e_tests/test_1.py::test_GitRepo_flyweight_monitoring_inode\nFAILED r2e_tests/test_1.py::test_GitRepo_invalid_path - TypeError: test_GitRe...\nFAILED r2e_tests/test_1.py::test_GitRepo_instance_from_existing - datalad.sup...\nFAILED r2e_tests/test_1.py::test_GitRepo_init_options - TypeError: test_GitRe...\nFAILED r2e_tests/test_1.py::test_init_fail_under_known_subdir - TypeError: te...\nFAILED r2e_tests/test_1.py::test_GitRepo_remove - TypeError: test_GitRepo_rem...\nFAILED r2e_tests/test_1.py::test_GitRepo_commit - TypeError: test_GitRepo_com...\nFAILED r2e_tests/test_1.py::test_GitRepo_dirty - TypeError: test_GitRepo_dirt...\nFAILED r2e_tests/test_1.py::test_submodule_deinit - TypeError: test_submodule...\nFAILED r2e_tests/test_1.py::test_get_submodules_parent_on_unborn_branch - Typ...\nFAILED r2e_tests/test_1.py::test_to_options - AssertionError: ['git', 'annex'...\nFAILED r2e_tests/test_1.py::test_optimized_cloning - TypeError: test_optimize...\nFAILED r2e_tests/test_1.py::test_GitRepo_gitignore - TypeError: test_GitRepo_...\nFAILED r2e_tests/test_1.py::test_GitRepo_set_remote_url - TypeError: test_Git...\nFAILED r2e_tests/test_1.py::test_gitattributes - TypeError: test_gitattribute...\nFAILED r2e_tests/test_1.py::test_get_hexsha_tag - TypeError: test_get_hexsha_...\nFAILED r2e_tests/test_1.py::test_get_tags - TypeError: test_get_tags() got mu...\nFAILED r2e_tests/test_1.py::test_get_commit_date - TypeError: test_get_commit...\nFAILED r2e_tests/test_1.py::test_fake_dates - TypeError: test_fake_dates() go...\nFAILED r2e_tests/test_1.py::test_custom_runner_protocol - TypeError: test_cus...\nFAILED r2e_tests/test_1.py::test_duecredit - TypeError: test_duecredit() got ...\nFAILED r2e_tests/test_1.py::test_GitRepo_get_revisions - TypeError: test_GitR...\nFAILED r2e_tests/test_1.py::test_gitrepo_add_to_git_with_annex_v7 - TypeError...\nFAILED r2e_tests/test_1.py::test_gitrepo_call_git_methods - TypeError: test_g...\n======== 23 failed, 3 passed, 2 skipped, 2 xfailed, 26 errors in 2.61s =========\n",
    "old_commit_res_stderr": ""
}
datalad/support/gitrepo.py datalad/support/tests/test_gitrepo.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `to_options` function rearranges command-line options, causing test mismatches

**Description:**
After updating the `to_options` function by importing code from GitPython, the order of generated command-line options has changed. This alteration leads to mismatches in tests that expect options in a specific sequence.

**Example Code:**
```python
def cmd_func(options=None):
    return ['git', 'annex', 'my_cmd'] + to_options(m='bla', force=True)

output = cmd_func()
print(output)
# Expected: ['git', 'annex', 'my_cmd', '--force', '-m', 'bla']
# Actual: ['git', 'annex', 'my_cmd', '-m', 'bla', '--force']
```

**Expected Behavior:**
The `to_options` function should generate command-line options in a consistent order, such as `--force` followed by `-m`, to match the expected command sequences in tests.

**Actual Behavior:**
The options are reordered, with `-m` appearing before `--force`, causing tests to fail due to the unexpected sequence.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/gitrepo.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to Git via GitPython

For further information on GitPython see http://gitpython.readthedocs.org/

"""

import re
import time
import os
import os.path as op
import warnings
from locale import getpreferredencoding


from itertools import chain
import logging
from collections import (
    OrderedDict,
    namedtuple
)
from os import linesep
from os.path import (
    join as opj,
    exists,
    normpath,
    isabs,
    commonprefix,
    relpath,
    realpath,
    dirname,
    basename,
    curdir,
    pardir,
    sep
)

import posixpath
from functools import wraps
from weakref import WeakValueDictionary

import git as gitpy
from gitdb.exc import BadName
from git.exc import (
    GitCommandError,
    NoSuchPathError,
    InvalidGitRepositoryError
)
from datalad.log import log_progress
from datalad.support.due import due, Doi

from datalad import ssh_manager
from datalad.cmd import (
    WitlessRunner,
    WitlessProtocol,
    GitRunner,
    BatchedCommand,
    run_gitcommand_on_file_list_chunks,
    StdOutErrCapture,
)
from datalad.config import (
    ConfigManager,
    _parse_gitconfig_dump
)

from datalad.consts import (
    ADJUSTED_BRANCH_EXPR,
)
from datalad.dochelpers import exc_str
import datalad.utils as ut
from datalad.utils import (
    Path,
    PurePosixPath,
    ensure_list,
    optional_args,
    on_windows,
    getpwd,
    posix_relpath,
    ensure_dir,
    generate_file_chunks,
    ensure_unicode,
    split_cmdline,
)

# imports from same module:
from .external_versions import external_versions
from .exceptions import (
    CommandError,
    DeprecatedError,
    FileNotInRepositoryError,
    GitIgnoreError,
    InvalidGitReferenceError,
    MissingBranchError,
    OutdatedExternalDependencyWarning,
    PathKnownToRepositoryError,
)
from .network import (
    RI,
    PathRI,
    is_ssh
)
from .path import get_parent_paths
from .repo import (
    PathBasedFlyweight,
    RepoInterface
)

#from git.objects.blob import Blob

# shortcuts
_curdirsep = curdir + sep
_pardirsep = pardir + sep


lgr = logging.getLogger('datalad.gitrepo')
_lgr_level = lgr.getEffectiveLevel()
if _lgr_level <= 2:
    from ..log import LoggerHelper
    # Let's also enable gitpy etc debugging
    gitpy_lgr = LoggerHelper(logtarget="git").get_initialized_logger()
    gitpy_lgr.setLevel(_lgr_level)
    gitpy_lgr.propagate = True

# Override default GitPython's DB backend to talk directly to git so it doesn't
# interfere with possible operations performed by gc/repack
default_git_odbt = gitpy.GitCmdObjectDB

# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used.
# Didn't work as expected on a first try. Probably there is a neatier way to
# log Exceptions from git commands.


def to_options(**kwargs):
    """Transform keyword arguments into a list of cmdline options

    Parameters
    ----------
    split_single_char_options: bool

    kwargs:

    Returns
    -------
    list
    """
    # TODO: borrow_docs!

    return gitpy.Git().transform_kwargs(**kwargs)


def _normalize_path(base_dir, path):
    """Helper to check paths passed to methods of this class.

    Checks whether `path` is beneath `base_dir` and normalizes it.
    Additionally paths are converted into relative paths with respect to
    `base_dir`, considering PWD in case of relative paths. This
    is intended to be used in repository classes, which means that
    `base_dir` usually will be the repository's base directory.

    Parameters
    ----------
    base_dir: str
        directory to serve as base to normalized, relative paths
    path: str
        path to be normalized

    Returns
    -------
    str:
        path, that is a relative path with respect to `base_dir`
    """
    if not path:
        return path

    base_dir = realpath(base_dir)  # realpath OK
    # path = normpath(path)
    # Note: disabled normpath, because it may break paths containing symlinks;
    # But we don't want to realpath relative paths, in case cwd isn't the
    # correct base.

    if isabs(path):
        # path might already be a symlink pointing to annex etc,
        # so realpath only its directory, to get "inline" with
        # realpath(base_dir) above
        path = opj(realpath(dirname(path)), basename(path))  # realpath OK
    # Executive decision was made to not do this kind of magic!
    #
    # elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:
    #     # If we are inside repository, rebuilt relative paths.
    #     path = opj(realpath(getpwd()), path)
    #
    # BUT with relative curdir/pardir start it would assume relative to curdir
    #
    elif path.startswith(_curdirsep) or path.startswith(_pardirsep):
        path = normpath(opj(realpath(getpwd()), path))  # realpath OK
    else:
        # We were called from outside the repo. Therefore relative paths
        # are interpreted as being relative to self.path already.
        return path

    if commonprefix([path, base_dir]) != base_dir:
        raise FileNotInRepositoryError(msg="Path outside repository: %s"
                                           % path, filename=path)

    return relpath(path, start=base_dir)


@optional_args
def normalize_path(func):
    """Decorator to provide unified path conversion for a single file

    Unlike normalize_paths, intended to be used for functions dealing with a
    single filename at a time

    Note
    ----
    This is intended to be used within the repository classes and therefore
    returns a class method!

    The decorated function is expected to take a path at
    first positional argument (after 'self'). Additionally the class `func`
    is a member of, is expected to have an attribute 'path'.
    """

    @wraps(func)
    def newfunc(self, file_, *args, **kwargs):
        file_new = _normalize_path(self.path, file_)
        return func(self, file_new, *args, **kwargs)

    return newfunc


@optional_args
def normalize_paths(func, match_return_type=True, map_filenames_back=False,
                    serialize=False):
    """Decorator to provide unified path conversions.

    Note
    ----
    This is intended to be used within the repository classes and therefore
    returns a class method!

    The decorated function is expected to take a path or a list of paths at
    first positional argument (after 'self'). Additionally the class `func`
    is a member of, is expected to have an attribute 'path'.

    Accepts either a list of paths or a single path in a str. Passes a list
    to decorated function either way, but would return based on the value of
    match_return_type and possibly input argument.

    If a call to the wrapped function includes normalize_path and it is False
    no normalization happens for that function call (used for calls to wrapped
    functions within wrapped functions, while possible CWD is within a
    repository)

    Parameters
    ----------
    match_return_type : bool, optional
      If True, and a single string was passed in, it would return the first
      element of the output (after verifying that it is a list of length 1).
      It makes easier to work with single files input.
    map_filenames_back : bool, optional
      If True and returned value is a dictionary, it assumes to carry entries
      one per file, and then filenames are mapped back to as provided from the
      normalized (from the root of the repo) paths
    serialize : bool, optional
      Loop through files giving only a single one to the function one at a time.
      This allows to simplify implementation and interface to annex commands
      which do not take multiple args in the same call (e.g. checkpresentkey)
    """

    @wraps(func)
    def newfunc(self, files, *args, **kwargs):

        normalize = _normalize_path if kwargs.pop('normalize_paths', True) \
            else lambda rpath, filepath: filepath

        if files:
            if isinstance(files, str) or not files:
                files_new = [normalize(self.path, files)]
                single_file = True
            elif isinstance(files, list):
                files_new = [normalize(self.path, path) for path in files]
                single_file = False
            else:
                raise ValueError("_files_decorator: Don't know how to handle "
                                 "instance of %s." % type(files))
        else:
            single_file = None
            files_new = []

        if map_filenames_back:
            def remap_filenames(out):
                """Helper to map files back to non-normalized paths"""
                if isinstance(out, dict):
                    assert(len(out) == len(files_new))
                    files_ = [files] if single_file else files
                    mapped = out.__class__()
                    for fin, fout in zip(files_, files_new):
                        mapped[fin] = out[fout]
                    return mapped
                else:
                    return out
        else:
            remap_filenames = lambda x: x

        if serialize:  # and not single_file:
            result = [
                func(self, f, *args, **kwargs)
                for f in files_new
            ]
        else:
            result = func(self, files_new, *args, **kwargs)

        if single_file is None:
            # no files were provided, nothing we can do really
            return result
        elif (result is None) or not match_return_type or not single_file:
            # If function doesn't return anything or no denormalization
            # was requested or it was not a single file
            return remap_filenames(result)
        elif single_file:
            if len(result) != 1:
                # Magic doesn't apply
                return remap_filenames(result)
            elif isinstance(result, (list, tuple)):
                return result[0]
            elif isinstance(result, dict) and tuple(result)[0] == files_new[0]:
                # assume that returned dictionary has files as keys.
                return tuple(result.values())[0]
            else:
                # no magic can apply
                return remap_filenames(result)
        else:
            return RuntimeError("should have not got here... check logic")

    return newfunc


def _remove_empty_items(list_):
    """Remove empty entries from list

    This is needed, since some functions of GitPython may convert
    an empty entry to '.', when used with a list of paths.

    Parameter:
    ----------
    list_: list of str

    Returns
    -------
    list of str
    """
    if not isinstance(list_, list):
        lgr.warning(
            "_remove_empty_items() called with non-list type: %s" % type(list_))
        return list_
    return [file_ for file_ in list_ if file_]


if "2.24.0" <= external_versions["cmd:git"] < "2.25.0":
    # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing
    # into untracked submodules when multiple pathspecs are given, returning
    # repositories that are deeper than the first level. This helper filters
    # these deeper levels out so that save_() doesn't fail trying to add them.
    #
    # This regression fixed with upstream's 072a231016 (2019-12-10).
    def _prune_deeper_repos(repos):
        firstlevel_repos = []
        prev = None
        for repo in sorted(repos):
            if not (prev and str(repo).startswith(prev)):
                prev = str(repo)
                firstlevel_repos.append(repo)
        return firstlevel_repos
else:
    def _prune_deeper_repos(repos):
        return repos


def Repo(*args, **kwargs):
    """Factory method around gitpy.Repo to consistently initiate with different
    backend
    """
    # TODO: This probably doesn't work as intended (or at least not as
    #       consistently as intended). gitpy.Repo could be instantiated by
    #       classmethods Repo.init or Repo.clone_from. In these cases 'odbt'
    #       would be needed as a parameter to these methods instead of the
    #       constructor.
    if 'odbt' not in kwargs:
        kwargs['odbt'] = default_git_odbt
    return gitpy.Repo(*args, **kwargs)


def guard_BadName(func):
    """A helper to guard against BadName exception

    Workaround for
    https://github.com/gitpython-developers/GitPython/issues/768
    also see https://github.com/datalad/datalad/issues/2550
    Let's try to precommit (to flush anything flushable) and do
    it again
    """

    @wraps(func)
    def wrapped(repo, *args, **kwargs):
        try:
            return func(repo, *args, **kwargs)
        except BadName:
            repo.precommit()
            return func(repo, *args, **kwargs)

    return wrapped


class GitProgress(WitlessProtocol):
    """Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """
    # inform super-class to capture stderr
    proc_err = True

    _num_op_codes = 10
    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \
        [1 << x for x in range(_num_op_codes)]
    STAGE_MASK = BEGIN | END
    OP_MASK = ~STAGE_MASK

    DONE_TOKEN = 'done.'
    TOKEN_SEPARATOR = ', '

    _known_ops = {
        COUNTING: ("Counting", "Objects"),
        ENUMERATING: ("Enumerating", "Objects"),
        COMPRESSING: ("Compressing", "Objects"),
        WRITING: ("Writing", "Objects"),
        RECEIVING: ("Receiving", "Objects"),
        RESOLVING: ("Resolving", "Deltas"),
        FINDING_SOURCES: ("Finding", "Sources"),
        CHECKING_OUT: ("Check out", "Things"),
    }

    __slots__ = ('_seen_ops', '_pbars', '_encoding')

    re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
    re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")

    def __init__(self, *args):
        super().__init__(*args)
        self._encoding = getpreferredencoding(do_setlocale=False)
        self._unprocessed = None

    def connection_made(self, transport):
        super().connection_made(transport)
        self._seen_ops = []
        self._pbars = set()

    def process_exited(self):
        # take down any progress bars that were not closed orderly
        for pbar_id in self._pbars:
            log_progress(
                lgr.info,
                pbar_id,
                'Finished',
            )
        super().process_exited()

    def pipe_data_received(self, fd, byts):
        # progress reports only come from stderr
        if fd != 2:
            # let the base class decide what to do with it
            super().pipe_data_received(fd, byts)
            return
        for line in byts.splitlines(keepends=True):
            # put any unprocessed content back in front
            line = self._unprocessed + line if self._unprocessed else line
            self._unprocessed = None
            if not self._parse_progress_line(line):
                # anything that doesn't look like a progress report
                # is retained and returned
                # in case of partial progress lines, this can lead to
                # leakage of progress info into the output, but
                # it is better to enable better (maybe more expensive)
                # subsequent filtering than hidding lines with
                # unknown, potentially important info
                lgr.debug('Non-progress stderr: %s', line)
                if line.endswith((b'\r', b'\n')):
                    # complete non-progress line, pass on
                    super().pipe_data_received(fd, line)
                else:
                    # an incomplete line, maybe the next batch completes
                    # it to become a recognizable progress report
                    self._unprocessed = line

    def _parse_progress_line(self, line):
        """Process a single line

        Parameters
        ----------
        line : bytes

        Returns
        -------
        bool
          Flag whether the line was recognized as a Git progress report.
        """
        # handle
        # Counting objects: 4, done.
        # Compressing objects:  50% (1/2)
        # Compressing objects: 100% (2/2)
        # Compressing objects: 100% (2/2), done.
        line = line.decode(self._encoding) if isinstance(line, bytes) else line
        if line.startswith(('warning:', 'error:', 'fatal:')):
            return False

        # find escape characters and cut them away - regex will not work with
        # them as they are non-ascii. As git might expect a tty, it will send them
        last_valid_index = None
        for i, c in enumerate(reversed(line)):
            if ord(c) < 32:
                # its a slice index
                last_valid_index = -i - 1
            # END character was non-ascii
        # END for each character in line
        if last_valid_index is not None:
            line = line[:last_valid_index]
        # END cut away invalid part
        line = line.rstrip()

        cur_count, max_count = None, None
        match = self.re_op_relative.match(line)
        if match is None:
            match = self.re_op_absolute.match(line)

        if not match:
            return False
        # END could not get match

        op_code = 0
        _remote, op_name, _percent, cur_count, max_count, message = match.groups()

        # get operation id
        if op_name == "Counting objects":
            op_code |= self.COUNTING
        elif op_name == "Compressing objects":
            op_code |= self.COMPRESSING
        elif op_name == "Writing objects":
            op_code |= self.WRITING
        elif op_name == 'Receiving objects':
            op_code |= self.RECEIVING
        elif op_name == 'Resolving deltas':
            op_code |= self.RESOLVING
        elif op_name == 'Finding sources':
            op_code |= self.FINDING_SOURCES
        elif op_name == 'Checking out files':
            op_code |= self.CHECKING_OUT
        elif op_name == 'Enumerating objects':
            op_code |= self.ENUMERATING
        else:
            # Note: On windows it can happen that partial lines are sent
            # Hence we get something like "CompreReceiving objects", which is
            # a blend of "Compressing objects" and "Receiving objects".
            # This can't really be prevented.
            lgr.debug(
                'Output line matched a progress report of an unknown type: %s',
                line)
            # TODO investigate if there is any chance that we might swallow
            # important info -- until them do not flag this line
            # as progress
            return False
        # END handle op code

        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)

        op_props = self._known_ops[op_code]

        # figure out stage
        if op_code not in self._seen_ops:
            self._seen_ops.append(op_code)
            op_code |= self.BEGIN
            log_progress(
                lgr.info,
                pbar_id,
                'Start {} {}'.format(
                    op_props[0].lower(),
                    op_props[1].lower(),
                ),
                label=op_props[0],
                unit=' {}'.format(op_props[1]),
                total=float(max_count) if max_count else None,
            )
            self._pbars.add(pbar_id)
        # END begin opcode

        if message is None:
            message = ''
        # END message handling

        done_progress = False
        message = message.strip()
        if message.endswith(self.DONE_TOKEN):
            op_code |= self.END
            message = message[:-len(self.DONE_TOKEN)]
            done_progress = True
        # END end message handling
        message = message.strip(self.TOKEN_SEPARATOR)

        if cur_count and max_count:
            log_progress(
                lgr.info,
                pbar_id,
                message,
                update=float(cur_count),
            )

        if done_progress:
            log_progress(
                lgr.info,
                pbar_id,
                'Finished {} {}'.format(
                    op_props[0].lower(),
                    op_props[1].lower(),
                ),
            )
            self._pbars.discard(pbar_id)
        return True


class StdOutCaptureWithGitProgress(GitProgress):
    proc_out = True


class FetchInfo(dict):
    """
    dict that carries results of a fetch operation of a single head

    Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """

    NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \
        FAST_FORWARD, ERROR = [1 << x for x in range(8)]

    _re_fetch_result = re.compile(r'^\s*(.) (\[?[\w\s\.$@]+\]?)\s+(.+) [-> ]+ ([^\s]+)(    \(.*\)?$)?')

    _flag_map = {
        '!': ERROR,
        '+': FORCED_UPDATE,
        '*': 0,
        '=': HEAD_UPTODATE,
        ' ': FAST_FORWARD,
        '-': TAG_UPDATE,
    }
    _operation_map = {
        NEW_TAG: 'new-tag',
        NEW_HEAD: 'new-branch',
        HEAD_UPTODATE: 'uptodate',
        TAG_UPDATE: 'tag-update',
        REJECTED: 'rejected',
        FORCED_UPDATE: 'forced-update',
        FAST_FORWARD: 'fast-forward',
        ERROR: 'error',
    }

    @classmethod
    def _from_line(cls, line):
        """Parse information from the given line as returned by git-fetch -v
        and return a new FetchInfo object representing this information.
        """
        match = cls._re_fetch_result.match(line)
        if match is None:
            raise ValueError("Failed to parse line: %r" % line)

        # parse lines
        control_character, operation, local_remote_ref, remote_local_ref, note = \
            match.groups()

        # parse flags from control_character
        flags = 0
        try:
            flags |= cls._flag_map[control_character]
        except KeyError:
            raise ValueError(
                "Control character %r unknown as parsed from line %r"
                % (control_character, line))
        # END control char exception handling

        # parse operation string for more info - makes no sense for symbolic refs,
        # but we parse it anyway
        old_commit = None
        if 'rejected' in operation:
            flags |= cls.REJECTED
        if 'new tag' in operation:
            flags |= cls.NEW_TAG
        if 'tag update' in operation:
            flags |= cls.TAG_UPDATE
        if 'new branch' in operation:
            flags |= cls.NEW_HEAD
        if '...' in operation or '..' in operation:
            split_token = '...'
            if control_character == ' ':
                split_token = split_token[:-1]
            old_commit = operation.split(split_token)[0]
        # END handle refspec

        return cls(
            ref=remote_local_ref.strip(),
            local_ref=local_remote_ref.strip(),
            # convert flag int into a list of operation labels
            operations=[
                cls._operation_map[o]
                for o in cls._operation_map.keys()
                if flags & o
            ],
            note=note,
            old_commit=old_commit,
        )


class PushInfo(dict):
    """dict that carries results of a push operation of a single head

    Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """
    NEW_TAG, NEW_HEAD, NO_MATCH, REJECTED, REMOTE_REJECTED, REMOTE_FAILURE, DELETED, \
        FORCED_UPDATE, FAST_FORWARD, UP_TO_DATE, ERROR = [1 << x for x in range(11)]

    _flag_map = {'X': NO_MATCH,
                 '-': DELETED,
                 '*': 0,
                 '+': FORCED_UPDATE,
                 ' ': FAST_FORWARD,
                 '=': UP_TO_DATE,
                 '!': ERROR}

    _operation_map = {
        NEW_TAG: 'new-tag',
        NEW_HEAD: 'new-branch',
        NO_MATCH: 'no-match',
        REJECTED: 'rejected',
        REMOTE_REJECTED: 'remote-rejected',
        REMOTE_FAILURE: 'remote-failure',
        DELETED: 'deleted',
        FORCED_UPDATE: 'forced-update',
        FAST_FORWARD: 'fast-forward',
        UP_TO_DATE: 'uptodate',
        ERROR: 'error',
    }

    @classmethod
    def _from_line(cls, line):
        """Create a new PushInfo instance as parsed from line which is expected to be like
            refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes"""
        control_character, from_to, summary = line.split('\t', 3)
        flags = 0

        # control character handling
        try:
            flags |= cls._flag_map[control_character]
        except KeyError:
            raise ValueError("Control character %r unknown as parsed from line %r" % (control_character, line))
        # END handle control character

        # from_to handling
        from_ref_string, to_ref_string = from_to.split(':')

        # commit handling, could be message or commit info
        old_commit = None
        if summary.startswith('['):
            if "[rejected]" in summary:
                flags |= cls.REJECTED
            elif "[remote rejected]" in summary:
                flags |= cls.REMOTE_REJECTED
            elif "[remote failure]" in summary:
                flags |= cls.REMOTE_FAILURE
            elif "[no match]" in summary:
                flags |= cls.ERROR
            elif "[new tag]" in summary:
                flags |= cls.NEW_TAG
            elif "[new branch]" in summary:
                flags |= cls.NEW_HEAD
            # uptodate encoded in control character
        else:
            # fast-forward or forced update - was encoded in control character,
            # but we parse the old and new commit
            split_token = "..."
            if control_character == " ":
                split_token = ".."
            old_sha, _new_sha = summary.split(' ')[0].split(split_token)
            # have to use constructor here as the sha usually is abbreviated
            old_commit = old_sha
        # END message handling

        return cls(
            from_ref=from_ref_string.strip(),
            to_ref=to_ref_string.strip(),
            # convert flag int into a list of operation labels
            operations=[
                cls._operation_map[o]
                for o in cls._operation_map.keys()
                if flags & o
            ],
            note=summary.strip(),
            old_commit=old_commit,
        )


# Compatibility kludge.  See GitRepo.get_submodules().
Submodule = namedtuple("Submodule", ["name", "path", "url"])


class GitRepo(RepoInterface, metaclass=PathBasedFlyweight):
    """Representation of a git repository

    """
    # We must check git config to have name and email set, but
    # should do it once
    _config_checked = False

    # Begin Flyweight:

    _unique_instances = WeakValueDictionary()

    def _flyweight_invalid(self):
        return not self.is_valid_git()

    @classmethod
    def _flyweight_reject(cls, id_, *args, **kwargs):
        # TODO:
        # This is a temporary approach. See PR # ...
        # create = kwargs.pop('create', None)
        # kwargs.pop('path', None)
        # if create and kwargs:
        #     # we have `create` plus options other than `path`
        #     return "Call to {0}() with args {1} and kwargs {2} conflicts " \
        #            "with existing instance {3}." \
        #            "This is likely to be caused by inconsistent logic in " \
        #            "your code." \
        #            "".format(cls, args, kwargs, cls._unique_instances[id_])
        pass

    # End Flyweight

    def __hash__(self):
        # the flyweight key is already determining unique instances
        # add the class name to distinguish from strings of a path
        return hash((self.__class__.__name__, self.__weakref__.key))

    # This is the least common denominator to claim that a user
    # used DataLad.
    # For now citing Zenodo's all (i.e., latest) version
    @due.dcite(Doi("10.5281/zenodo.808846"),
               # override path since there is no need ATM for such details
               path="datalad",
               description="DataLad - Data management and distribution platform")
    def __init__(self, path, url=None, runner=None, create=True,
                 git_opts=None, repo=None, fake_dates=False,
                 create_sanity_checks=True,
                 **kwargs):
        """Creates representation of git repository at `path`.

        Can also be used to create a git repository at `path`.

        Parameters
        ----------
        path: str
          path to the git repository; In case it's not an absolute path,
          it's relative to PWD
        url: str, optional
          DEPRECATED -- use .clone() class method
          url to the to-be-cloned repository. Requires a valid git url
          according to:
          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .
        create: bool, optional
          if true, creates a git repository at `path` if there is none. Also
          creates `path`, if it doesn't exist.
          If set to false, an exception is raised in case `path` doesn't exist
          or doesn't contain a git repository.
        repo: git.Repo, optional
          GitPython's Repo instance to (re)use if provided
        create_sanity_checks: bool, optional
          Whether to perform sanity checks during initialization (when
          `create=True` and target path is not a valid repo already), such as
          that new repository is not created in the directory where git already
          tracks some files.
        kwargs:
          keyword arguments serving as additional options to the git-init
          command. Therefore, it makes sense only if called with `create`.

          Generally, this way of passing options to the git executable is
          (or will be) used a lot in this class. It's a transformation of
          python-style keyword arguments (or a `dict`) to command line arguments,
          provided by GitPython.

          A single character keyword will be prefixed by '-', multiple characters
          by '--'. An underscore in the keyword becomes a dash. The value of the
          keyword argument is used as the value for the corresponding command
          line argument. Assigning a boolean creates a flag.

          Examples:
          no_commit=True => --no-commit
          C='/my/path'   => -C /my/path

        """

        # BEGIN Repo validity test
        # We want to fail early for tests, that would be performed a lot. In particular this is about
        # GitRepo.is_valid_repo. We would use the latter to decide whether or not to call GitRepo() only for
        # __init__ to then test the same things again. If we fail early we can save the additional test from outer
        # scope.
        self.path = path

        # Note, that the following three path objects are used often and therefore
        # are stored for performance. Path object creation comes with a cost. Most noteably,
        # this is used for validity checking of the repository.
        self.pathobj = ut.Path(self.path)
        self.dot_git = self.pathobj / '.git'
        # Read a potential .git file in order to not do that over and over again, when testing is_valid_git() etc.
        # TODO: There's still some code duplication with static method GitRepo.get_git_dir()
        #       However, it's returning relative path. So, the logic in usage needs to be unified in order to melt both
        #       pieces.
        if self.dot_git.is_file():
            with self.dot_git.open() as f:
                line = f.readline()
                if line.startswith("gitdir: "):
                    self.dot_git = self.pathobj / line[7:].strip()
                else:
                    raise InvalidGitRepositoryError("Invalid .git file")

        elif self.dot_git.is_symlink():
            self.dot_git = self.dot_git.resolve()

        self._valid_git_test_path = self.dot_git / 'HEAD'
        _valid_repo = self.is_valid_git()

        do_create = False
        if create and not _valid_repo:
            if repo is not None:
                # `repo` passed with `create`, which doesn't make sense
                raise TypeError("argument 'repo' must not be used with 'create'")
            do_create = True
        else:
            # Note: We used to call gitpy.Repo(path) here, which potentially
            # raised NoSuchPathError or InvalidGitRepositoryError. This is
            # used by callers of GitRepo.__init__() to detect whether we have a
            # valid repo at `path`. Now, with switching to lazy loading property
            # `repo`, we detect those cases without instantiating a
            # gitpy.Repo().

            if not exists(path):
                raise NoSuchPathError(path)
            if not _valid_repo:
                raise InvalidGitRepositoryError(path)
        # END Repo validity test

        if url is not None:
            raise DeprecatedError(
                new=".clone() class method",
                version="0.5.0",
                msg="RF: url passed to init()"
            )

        # So that we "share" control paths with git/git-annex
        if ssh_manager:
            ssh_manager.assure_initialized()

        # note: we may also want to distinguish between a path to the worktree
        # and the actual repository

        # Could be used to e.g. disable automatic garbage and autopacking
        # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0']
        self._GIT_COMMON_OPTIONS = []
        # actually no need with default GitPython db backend not in memory
        # default_git_odbt but still allows for faster testing etc.
        # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = []

        if git_opts is None:
            git_opts = {}
        if kwargs:
            git_opts.update(kwargs)

        self.cmd_call_wrapper = runner or GitRunner(cwd=self.path)
        self._repo = repo
        self._cfg = None

        if do_create:  # we figured it out ealier
            self._repo = self._create_empty_repo(path,
                                                 create_sanity_checks, **git_opts)

        # inject git options into GitPython's git call wrapper:
        # Note: `None` currently can happen, when Runner's protocol prevents
        # calls above from being actually executed (DryRunProtocol)
        if self._repo is not None:
            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS

        # with DryRunProtocol path might still not exist
        if exists(self.path):
            self.inode = os.stat(self.path).st_ino
        else:
            self.inode = None

        if fake_dates:
            self.configure_fake_dates()
        # Set by fake_dates_enabled to cache config value across this instance.
        self._fake_dates_enabled = None

    def _create_empty_repo(self, path, sanity_checks=True, **kwargs):
        if not op.lexists(path):
            os.makedirs(path)
        elif sanity_checks and external_versions['cmd:git'] < '2.14.0':
            warnings.warn(
                "Your git version (%s) is too old, we will not safe-guard "
                "against creating a new repository under already known to git "
                "subdirectory" % external_versions['cmd:git'],
                OutdatedExternalDependencyWarning
            )
        elif sanity_checks:
            # Verify that we are not trying to initialize a new git repository
            # under a directory some files of which are already tracked by git
            # use case: https://github.com/datalad/datalad/issues/3068
            try:
                stdout, _ = self._git_custom_command(
                    None, ['git', 'ls-files'], cwd=path, expect_fail=True
                )
                if stdout:
                    raise PathKnownToRepositoryError(
                        "Failing to initialize new repository under %s where "
                        "following files are known to a repository above: %s"
                        % (path, stdout)
                    )
            except CommandError:
                # assume that all is good -- we are not under any repo
                pass

        cmd = ['git', 'init']
        cmd.extend(kwargs.pop('_from_cmdline_', []))
        cmd.extend(to_options(**kwargs))
        lgr.debug(
            "Initialize empty Git repository at '%s'%s",
            path,
            ' %s' % cmd[2:] if cmd[2:] else '')

        try:
            stdout, stderr = self._git_custom_command(
                None,
                cmd,
                cwd=path,
                log_stderr=True,
                log_stdout=True,
                log_online=False,
                expect_stderr=False,
                shell=False,
                # we don't want it to scream on stdout
                expect_fail=True)
        except CommandError as exc:
            lgr.error(exc_str(exc))
            raise
        # we want to return None and have lazy eval take care of
        # the rest
        return

    @property
    def repo(self):
        # with DryRunProtocol path not exist
        if exists(self.path):
            inode = os.stat(self.path).st_ino
        else:
            inode = None
        if self.inode != inode:

            # TODO: - what if we never had any inode? possible?
            #       - also: call to self._repo and only afterwards checking whether it's None seems strange

            # reset background processes invoked by GitPython:
            # it can be that we never got to instantiate a GitPython repo instance
            if self._repo:
                self._repo.git.clear_cache()
            self.inode = inode

        if self._repo is None:
            # Note, that this may raise GitCommandError, NoSuchPathError,
            # InvalidGitRepositoryError:
            self._repo = self.cmd_call_wrapper(
                Repo,
                self.path)
            lgr.log(8, "Using existing Git repository at %s", self.path)

        # inject git options into GitPython's git call wrapper:
        # Note: `None` currently can happen, when Runner's protocol prevents
        # call of Repo(path) above from being actually executed (DryRunProtocol)
        if self._repo is not None:
            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS

        return self._repo

    @classmethod
    def clone(cls, url, path, *args, clone_options=None, **kwargs):
        """Clone url into path

        Provides workarounds for known issues (e.g.
        https://github.com/datalad/datalad/issues/785)

        Parameters
        ----------
        url : str
        path : str
        clone_options : dict
          Key/value pairs of arbitrary options that will be passed on to the
          underlying call to `git-clone`.
        expect_fail : bool
          Whether expect that command might fail, so error should be logged then
          at DEBUG level instead of ERROR
        """

        if 'repo' in kwargs:
            raise TypeError("argument 'repo' conflicts with cloning")
            # TODO: what about 'create'?

        expect_fail = kwargs.pop('expect_fail', False)
        # fail early on non-empty target:
        from os import listdir
        if exists(path) and listdir(path):
            # simulate actual GitCommandError:
            lgr.warning("destination path '%s' already exists and is not an "
                        "empty directory." % path)
            raise GitCommandError(
                ['git', 'clone', '-v', url, path],
                128,
                "fatal: destination path '%s' already exists and is not an "
                "empty directory." % path)
        else:
            # protect against cloning into existing and obviously dangling
            # instance for that location
            try:
                del cls._unique_instances[path]
            except KeyError:
                # didn't exist - all fine
                pass

        # Massage URL
        url_ri = RI(url) if not isinstance(url, RI) else url
        if not on_windows:
            # if we are on windows, the local path of a URL
            # would not end up being a proper local path and cloning
            # would fail. Don't try to be smart and just pass the
            # URL along unmodified

            # try to get a local path from `url`:
            try:
                url = url_ri.localpath
                url_ri = RI(url)
            except ValueError:
                pass

        if is_ssh(url_ri):
            ssh_manager.get_connection(url).open()
        else:
            if isinstance(url_ri, PathRI):
                # expand user, because execution not going through a shell
                # doesn't work well otherwise
                new_url = os.path.expanduser(url)
                if url != new_url:
                    lgr.info("Expanded source path to %s from %s", new_url, url)
                    url = new_url

        fix_annex = None
        ntries = 5  # 3 is not enough for robust workaround
        for trial in range(ntries):
            try:
                lgr.debug("Git clone from {0} to {1}".format(url, path))

                out, err = WitlessRunner(
                    env=GitRunner.get_git_environ_adjusted()).run(
                        ['git', 'clone', '--progress', url, path] \
                        + (to_options(**clone_options)
                           if clone_options else []),
                        protocol=GitProgress,
                )
                # fish out non-critical warnings by git-clone
                # (empty repo clone, etc.), all other content is logged
                # by the progress helper to 'debug'
                for errline in err.splitlines():
                    if errline.startswith('warning:'):
                        lgr.warning(errline[8:].strip())
                lgr.debug("Git clone completed")
                break
            except CommandError as e:
                # log here but let caller decide what to do
                e_str = exc_str(e)
                # see https://github.com/datalad/datalad/issues/785
                if re.search("Request for .*aborted.*Unable to find", str(e),
                             re.DOTALL) \
                        and trial < ntries - 1:
                    lgr.info(
                        "Hit a known issue with Git (see GH#785). Trial #%d, "
                        "retrying",
                        trial)
                    continue
                    (lgr.debug if expect_fail else lgr.error)(e_str)

                if "Clone succeeded, but checkout failed." in str(e):
                    fix_annex = e
                    break

                raise

        # get ourselves a repository instance
        gr = cls(path, *args, **kwargs)
        if fix_annex:
            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then
            # would need to import our own subclass
            if hasattr(gr, 'is_valid_annex'):
                lgr.warning("Experienced issues while cloning. "
                            "Trying to fix it, using git-annex-fsck.")
                if not gr.is_initialized():
                    gr._init()
                gr.fsck()
            else:
                lgr.warning("Experienced issues while cloning: %s", exc_str(fix_annex))
        return gr

    def __del__(self):
        # unbind possibly bound ConfigManager, to prevent all kinds of weird
        # stalls etc
        self._cfg = None
        # Make sure to flush pending changes, especially close batch processes
        # (internal `git cat-file --batch` by GitPython)
        try:
            if getattr(self, '_repo', None) is not None and exists(self.path):
                # gc might be late, so the (temporary)
                # repo doesn't exist on FS anymore
                self._repo.git.clear_cache()
                # We used to write out the index to flush GitPython's
                # state... but such unconditional write is really a workaround
                # and does not play nice with read-only operations - permission
                # denied etc. So disabled 
                #if exists(opj(self.path, '.git')):  # don't try to write otherwise
                #    self.repo.index.write()
        except (InvalidGitRepositoryError, AttributeError):
            # might have being removed and no longer valid or attributes unbound
            pass

    def __repr__(self):
        return "<GitRepo path=%s (%s)>" % (self.path, type(self))

    def __eq__(self, obj):
        """Decides whether or not two instances of this class are equal.

        This is done by comparing the base repository path.
        """
        return self.path == obj.path

    def is_valid_git(self):
        """Returns whether the underlying repository appears to be still valid

        Note, that this almost identical to the classmethod is_valid_repo(). However,
        if we are testing an existing instance, we can save Path object creations. Since this testing
        is done a lot, this is relevant. Creation of the Path objects in is_valid_repo() takes nearly half the time of
        the entire function.

        Also note, that this method is bound to an instance but still class-dependent, meaning that a subclass
        cannot simply overwrite it. This is particularly important for the call from within __init__(),
        which in turn is called by the subclasses' __init__. Using an overwrite would lead to the wrong thing being
        called.
        """

        return self.dot_git.exists() and (
                not self.dot_git.is_dir() or self._valid_git_test_path.exists()
        )

    @classmethod
    def is_valid_repo(cls, path):
        """Returns if a given path points to a git repository"""
        if not isinstance(path, Path):
            path = Path(path)
        dot_git_path = path / '.git'

        # the aim here is to have this test as cheap as possible, because
        # it is performed a lot
        # recognize two things as good-enough indicators of a present
        # repo: 1) a non-empty .git directory (#3473)
        #          NOTE: It's actually faster (and more accurate) to test for existence of a particular subpath.
        #                This should be something that's there right after git-init. Going for .git/HEAD ATM.
        #
        #                In [11]: %timeit path.exists() and (not path.is_dir() or head_path.exists())
        #                4.93 µs ± 34.8 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
        #                In [12]: %timeit path.exists() and (not path.is_dir() or any(path.iterdir()))
        #                12.8 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
        #
        #   and 2) a pointer file or symlink

        return dot_git_path.exists() and (
            not dot_git_path.is_dir() or (dot_git_path / 'HEAD').exists()
        )

    @staticmethod
    def get_git_dir(repo):
        """figure out a repo's gitdir

        '.git' might be a  directory, a symlink or a file

        Note
        ----
        Please try using GitRepo.dot_git instead! That one's not static, but it's cheaper and you should avoid
        not having an instance of a repo you're working on anyway. Note, that the property in opposition to this method
        returns an absolute path.


        Parameters
        ----------
        repo: path or Repo instance
          currently expected to be the repos base dir

        Returns
        -------
        str
          relative path to the repo's git dir; So, default would be ".git"
        """
        if hasattr(repo, 'path'):
            # repo instance like given
            repo = repo.path
        dot_git = op.join(repo, ".git")
        if not op.exists(dot_git):
            raise RuntimeError("Missing .git in %s." % repo)
        elif op.islink(dot_git):
            git_dir = os.readlink(dot_git)
        elif op.isdir(dot_git):
            git_dir = ".git"
        elif op.isfile(dot_git):
            with open(dot_git) as f:
                git_dir = f.readline()
                if git_dir.startswith("gitdir:"):
                    git_dir = git_dir[7:]
                git_dir = git_dir.strip()

        return git_dir

    @property
    def config(self):
        """Get an instance of the parser for the persistent repository
        configuration.

        Note: This allows to also read/write .datalad/config,
        not just .git/config

        Returns
        -------
        ConfigManager
        """
        if self._cfg is None:
            # associate with this dataset and read the entire config hierarchy
            self._cfg = ConfigManager(dataset=self, source='any')
        return self._cfg

    def is_with_annex(self):
        """Report if GitRepo (assumed) has (remotes with) a git-annex branch
        """
        return any(
            b['refname:strip=2'] == 'git-annex' or b['refname:strip=2'].endswith('/git-annex')
            for b in self.for_each_ref_(fields='refname:strip=2', pattern=['refs/heads', 'refs/remotes'])
        )

    @classmethod
    def get_toppath(cls, path, follow_up=True, git_options=None):
        """Return top-level of a repository given the path.

        Parameters
        -----------
        follow_up : bool
          If path has symlinks -- they get resolved by git.  If follow_up is
          True, we will follow original path up until we hit the same resolved
          path.  If no such path found, resolved one would be returned.
        git_options: list of str
          options to be passed to the git rev-parse call

        Return None if no parent directory contains a git repository.
        """
        cmd = ['git']
        if git_options:
            cmd.extend(git_options)
        cmd += ["rev-parse", "--show-toplevel"]
        try:
            toppath, err = GitRunner().run(
                cmd,
                cwd=path,
                log_stdout=True, log_stderr=True,
                expect_fail=True, expect_stderr=True)
            toppath = toppath.rstrip('\n\r')
        except CommandError:
            return None
        except OSError:
            toppath = GitRepo.get_toppath(dirname(path), follow_up=follow_up,
                                          git_options=git_options)

        if follow_up:
            path_ = path
            path_prev = ""
            while path_ and path_ != path_prev:  # on top /.. = /
                if realpath(path_) == toppath:
                    toppath = path_
                    break
                path_prev = path_
                path_ = dirname(path_)

        return toppath

    @normalize_paths
    def add(self, files, git=True, git_options=None, update=False):
        """Adds file(s) to the repository.

        Parameters
        ----------
        files: list
          list of paths to add
        git: bool
          somewhat ugly construction to be compatible with AnnexRepo.add();
          has to be always true.
        update: bool
          --update option for git-add. From git's manpage:
           Update the index just where it already has an entry matching
           <pathspec>. This removes as well as modifies index entries to match
           the working tree, but adds no new files.

           If no <pathspec> is given when --update option is used, all tracked
           files in the entire working tree are updated (old versions of Git
           used to limit the update to the current directory and its
           subdirectories).

        Returns
        -------
        list
          Of status dicts.
        """
        # under all circumstances call this class' add_ (otherwise
        # AnnexRepo.add would go into a loop
        return list(GitRepo.add_(self, files, git=git, git_options=git_options,
                    update=update))

    def add_(self, files, git=True, git_options=None, update=False):
        """Like `add`, but returns a generator"""
        # TODO: git_options is used as options for the git-add here,
        # instead of options to the git executable => rename for consistency

        if not git:
            lgr.warning(
                'GitRepo.add() called with git=%s, this should not happen',
                git)
            git = True

        # there is no other way then to collect all files into a list
        # at this point, because we need to pass them at once to a single
        # `git add` call
        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]

        if not (files or git_options or update):
            # wondering why just a warning? in cmdline this is also not an error
            lgr.warning("add was called with empty file list and no options.")
            return

        try:
            # without --verbose git 2.9.3  add does not return anything
            add_out = self._git_custom_command(
                files,
                # Set annex.largefiles to prevent storing files in annex when
                # GitRepo() is instantiated with a v6+ annex repo.
                ['git', '-c', 'annex.largefiles=nothing', 'add'] +
                ensure_list(git_options) +
                to_options(update=update) + ['--verbose']
            )
            # get all the entries
            for o in self._process_git_get_output(*add_out):
                yield o
            # Note: as opposed to git cmdline, force is True by default in
            #       gitpython, which would lead to add things, that are
            #       ignored or excluded otherwise
            # 2. Note: There is an issue with globbing (like adding '.'),
            #       which apparently doesn't care for 'force' and therefore
            #       adds '.git/...'. May be it's expanded at the wrong
            #       point in time or sth. like that.
            # For now, use direct call to git add.
            #self.cmd_call_wrapper(self.repo.index.add, files, write=True,
            #                      force=False)
            # TODO: May be make use of 'fprogress'-option to indicate
            # progress
            # But then, we don't have it for git-annex add, anyway.
            #
            # TODO: Is write=True a reasonable way to do it?
            # May be should not write until success of operation is
            # confirmed?
            # What's best in case of a list of files?
        except OSError as e:
            lgr.error("add: %s" % e)
            raise

        # Make sure return value from GitRepo is consistent with AnnexRepo
        # currently simulating similar return value, assuming success
        # for all files:
        # TODO: Make return values consistent across both *Repo classes!
        return

    @staticmethod
    def _process_git_get_output(stdout, stderr=None):
        """Given both outputs (stderr is ignored atm) of git add - process it

        Primarily to centralize handling in both indirect annex and direct
        modes when ran through proxy
        """
        return [{u'file': f, u'success': True}
                for f in re.findall("'(.*)'[\n$]", ensure_unicode(stdout))]

    @normalize_paths(match_return_type=False)
    def remove(self, files, recursive=False, **kwargs):
        """Remove files.

        Calls git-rm.

        Parameters
        ----------
        files: str
          list of paths to remove
        recursive: False
          whether to allow recursive removal from subdirectories
        kwargs:
          see `__init__`

        Returns
        -------
        [str]
          list of successfully removed files.
        """

        files = _remove_empty_items(files)

        if recursive:
            kwargs['r'] = True
        stdout, stderr = self._git_custom_command(
            files, ['git', 'rm'] + to_options(**kwargs))

        # output per removed file is expected to be "rm 'PATH'":
        return [line.strip()[4:-1] for line in stdout.splitlines()]

        #return self.repo.git.rm(files, cached=False, **kwargs)

    def precommit(self):
        """Perform pre-commit maintenance tasks
        """
        # All GitPython commands should take care about flushing index
        # whenever they modify it, so we would not care to do anything
        # if self.repo is not None and exists(opj(self.path, '.git')):  # don't try to write otherwise:
        #     # flush possibly cached in GitPython changes to index:
        #     # if self.repo.git:
        #     #     sys.stderr.write("CLEARING\n")
        #     #     self.repo.git.clear_cache()
        #     self.repo.index.write()

        # Close batched by GitPython git processes etc
        # Ref: https://github.com/gitpython-developers/GitPython/issues/718
        self.repo.__del__()
        pass

    @staticmethod
    def _get_prefixed_commit_msg(msg):
        DATALAD_PREFIX = "[DATALAD]"
        return DATALAD_PREFIX if not msg else "%s %s" % (DATALAD_PREFIX, msg)

    def for_each_ref_(self, fields=('objectname', 'objecttype', 'refname'),
                      pattern=None, points_at=None, sort=None, count=None,
                      contains=None):
        """Wrapper for `git for-each-ref`

        Please see manual page git-for-each-ref(1) for a complete overview
        of its functionality. Only a subset of it is supported by this
        wrapper.

        Parameters
        ----------
        fields : iterable or str
          Used to compose a NULL-delimited specification for for-each-ref's
          --format option. The default field list reflects the standard
          behavior of for-each-ref when the --format option is not given.
        pattern : list or str, optional
          If provided, report only refs that match at least one of the given
          patterns.
        points_at : str, optional
          Only list refs which points at the given object.
        sort : list or str, optional
          Field name(s) to sort-by. If multiple fields are given, the last one
          becomes the primary key. Prefix any field name with '-' to sort in
          descending order.
        count : int, optional
          Stop iteration after the given number of matches.
        contains : str, optional
          Only list refs which contain the specified commit.

        Yields
        ------
        dict with items matching the given `fields`

        Raises
        ------
        ValueError
          if no `fields` are given

        RuntimeError
          if `git for-each-ref` returns a record where the number of
          properties does not match the number of `fields`
        """
        if not fields:
            raise ValueError('no `fields` provided, refuse to proceed')
        fields = ensure_list(fields)
        cmd = [
            "git",
            "for-each-ref",
            "--format={}".format(
                '%00'.join(
                    '%({})'.format(f) for f in fields)),
        ]
        if points_at:
            cmd.append('--points-at={}'.format(points_at))
        if contains:
            cmd.append('--contains={}'.format(contains))
        if sort:
            for k in ensure_list(sort):
                cmd.append('--sort={}'.format(k))
        if pattern:
            cmd += ensure_list(pattern)
        if count:
            cmd.append('--count={:d}'.format(count))

        out, _ = self._git_custom_command(None, cmd)
        for line in out.splitlines():
            props = line.split('\0')
            if len(fields) != len(props):
                raise RuntimeError(
                    'expected fields {} from git-for-each-ref, but got: {}'.format(
                        fields, props))
            yield dict(zip(fields, props))

    def configure_fake_dates(self):
        """Configure repository to use fake dates.
        """
        lgr.debug("Enabling fake dates")
        self.config.set("datalad.fake-dates", "true")

    @property
    def fake_dates_enabled(self):
        """Is the repository configured to use fake dates?
        """
        if self._fake_dates_enabled is None:
            self._fake_dates_enabled = \
                self.config.getbool('datalad', 'fake-dates', default=False)
        return self._fake_dates_enabled

    def add_fake_dates(self, env):
        """Add fake dates to `env`.

        Parameters
        ----------
        env : dict or None
            Environment variables.

        Returns
        -------
        A dict (copied from env), with date-related environment
        variables for git and git-annex set.
        """
        env = (env if env is not None else os.environ).copy()
        # Note: Use _git_custom_command here rather than repo.git.for_each_ref
        # so that we use annex-proxy in direct mode.
        last_date = list(self.for_each_ref_(
            fields='committerdate:raw',
            count=1,
            pattern='refs/heads',
            sort="-committerdate",
        ))

        if last_date:
            # Drop the "contextual" timezone, leaving the unix timestamp.  We
            # avoid :unix above because it wasn't introduced until Git v2.9.4.
            last_date = last_date[0]['committerdate:raw'].split()[0]
            seconds = int(last_date)
        else:
            seconds = self.config.obtain("datalad.fake-dates-start")
        seconds_new = seconds + 1
        date = "@{} +0000".format(seconds_new)

        lgr.debug("Setting date to %s",
                  time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                time.gmtime(seconds_new)))

        env["GIT_AUTHOR_DATE"] = date
        env["GIT_COMMITTER_DATE"] = date
        env["GIT_ANNEX_VECTOR_CLOCK"] = str(seconds_new)

        return env

    def commit(self, msg=None, options=None, _datalad_msg=False, careless=True,
               files=None, date=None, index_file=None):
        """Commit changes to git.

        Parameters
        ----------
        msg: str, optional
          commit-message
        options: list of str, optional
          cmdline options for git-commit
        _datalad_msg: bool, optional
          To signal that commit is automated commit by datalad, so
          it would carry the [DATALAD] prefix
        careless: bool, optional
          if False, raise when there's nothing actually committed;
          if True, don't care
        files: list of str, optional
          path(s) to commit
        date: str, optional
          Date in one of the formats git understands
        index_file: str, optional
          An alternative index to use
        """

        self.precommit()

        # assemble commandline
        cmd = ['git', 'commit']
        options = ensure_list(options)

        if date:
            options += ["--date", date]

        if not msg:
            msg = 'Recorded changes'
            _datalad_msg = True

        if _datalad_msg:
            msg = self._get_prefixed_commit_msg(msg)

        options += ["-m", msg]
        cmd.extend(options)

        # set up env for commit
        env = GitRunner.get_git_environ_adjusted()
        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)
        if index_file:
            env['GIT_INDEX_FILE'] = index_file

        lgr.debug("Committing via direct call of git: %s" % cmd)

        file_chunks = generate_file_chunks(files, cmd) if files else [[]]

        runner = WitlessRunner(cwd=self.path, env=env)

        try:
            for i, chunk in enumerate(file_chunks):
                cur_cmd = cmd + (
                    # if this is an explicit dry-run, there is no point in
                    # amending, because no commit was ever made
                    # otherwise, amend the first commit, and prevent
                    # leaving multiple commits behind
                    ['--amend', '--no-edit']
                    if i > 0 and '--dry-run' not in cmd
                    else []
                ) + ['--'] + chunk
                runner.run(
                    cur_cmd,
                    protocol=StdOutErrCapture,
                    stdin=None,
                )
        except CommandError as e:
            # real errors first
            if "did not match any file(s) known to git" in e.stderr:
                raise FileNotInRepositoryError(
                    cmd=e.cmd,
                    msg="File(s) unknown to git",
                    code=e.code,
                    filename=linesep.join([
                        l for l in e.stderr.splitlines()
                        if l.startswith("error: pathspec")
                    ])
                )
            # behavior choices now
            elif not careless:
                # not willing to compromise at all
                raise
            elif 'nothing to commit' in e.stdout:
                lgr.debug("nothing to commit in %s. Ignored.", self)
            elif 'no changes added to commit' in e.stdout or \
                    'nothing added to commit' in e.stdout:
                lgr.debug("no changes added to commit in %s. Ignored.", self)
            else:
                raise

    # TODO usage is primarily in the tests, consider making a test helper and
    # remove from GitRepo API
    def get_indexed_files(self):
        """Get a list of files in git's index

        Returns
        -------
        list
            list of paths rooting in git's base dir
        """

        return [
            str(r.relative_to(self.pathobj))
            for r in self.get_content_info(
                paths=None, ref=None, untracked='no', eval_file_type=False)
        ]

    def format_commit(self, fmt, commitish=None):
        """Return `git show` output for `commitish`.

        Parameters
        ----------
        fmt : str
            A format string accepted by `git show`.
        commitish: str, optional
          Any commit identifier (defaults to "HEAD").

        Returns
        -------
        str or, if there are not commits yet, None.
        """
        cmd = ['git', 'show', '-z', '--no-patch', '--format=' + fmt]
        if commitish is not None:
            cmd.append(commitish + "^{commit}")
        # make sure Git takes our argument as a revision
        cmd.append('--')
        try:
            stdout, stderr = self._git_custom_command(
                '', cmd, expect_stderr=True, expect_fail=True)
        except CommandError as e:
            if 'bad revision' in e.stderr:
                raise ValueError("Unknown commit identifier: %s" % commitish)
            elif 'does not have any commits yet' in e.stderr:
                return None
            else:
                raise e
        # This trailing null is coming from the -z above, which avoids the
        # newline that Git would append to the output. We could drop -z and
        # strip the newline directly, but then we'd have to worry about
        # compatibility across platforms.
        return stdout.rsplit("\0", 1)[0]

    def get_hexsha(self, commitish=None, short=False):
        """Return a hexsha for a given commitish.

        Parameters
        ----------
        commitish : str, optional
          Any identifier that refers to a commit (defaults to "HEAD").
        short : bool, optional
          Return the abbreviated form of the hexsha.

        Returns
        -------
        str or, if there are not commits yet, None.
        """
        stdout = self.format_commit("%{}".format('h' if short else 'H'),
                                    commitish)
        if stdout is not None:
            stdout = stdout.splitlines()
            assert(len(stdout) == 1)
            return stdout[0]

    @normalize_paths(match_return_type=False)
    def get_last_commit_hexsha(self, files):
        """Return the hash of the last commit the modified any of the given
        paths"""
        try:
            commit = self.call_git(
                ['rev-list', '-n1', 'HEAD'],
                files=files,
                expect_fail=True,
            )
            commit = commit.strip()
            return commit if commit else None
        except CommandError:
            if self.get_hexsha() is None:
                # unborn branch, don't freak out
                return None
            raise

    def get_revisions(self, revrange=None, fmt="%H", options=None):
        """Return list of revisions in `revrange`.

        Parameters
        ----------
        revrange : str or list of str or None, optional
            Revisions or revision ranges to walk. If None, revision defaults to
            HEAD unless a revision-modifying option like `--all` or
            `--branches` is included in `options`.
        fmt : string, optional
            Format accepted by `--format` option of `git log`. This should not
            contain new lines because the output is split on new lines.
        options : list of str, optional
            Options to pass to `git log`.  This should not include `--format`.

        Returns
        -------
        List of revisions (str), formatted according to `fmt`.
        """
        if revrange is None:
            revrange = []
        elif isinstance(revrange, str):
            revrange = [revrange]

        cmd = ["git", "log", "--format={}".format(fmt)]
        cmd.extend((options or []) + revrange + ["--"])
        try:
            stdout, _ = self._git_custom_command(None, cmd, expect_fail=True)
        except CommandError as e:
            if "does not have any commits" in e.stderr:
                return []
            raise
        return stdout.splitlines()

    def commit_exists(self, commitish):
        """Does `commitish` exist in the repo?

        Parameters
        ----------
        commitish : str
            A commit or an object that can be dereferenced to one.

        Returns
        -------
        bool
        """
        # Note: The peeling operator "^{commit}" is required so that rev-parse
        # doesn't succeed if passed a full hexsha that is valid but doesn't
        # exist.
        return self.call_git_success(
            ["rev-parse", "--verify", commitish + "^{commit}"])

    def get_merge_base(self, commitishes):
        """Get a merge base hexsha

        Parameters
        ----------
        commitishes: str or list of str
          List of commitishes (branches, hexshas, etc) to determine the merge
          base of. If a single value provided, returns merge_base with the
          current branch.

        Returns
        -------
        str or None
          If no merge-base for given commits, or specified treeish doesn't
          exist, None returned
        """
        if isinstance(commitishes, str):
            commitishes = [commitishes]
        if not commitishes:
            raise ValueError("Provide at least a single value")
        elif len(commitishes) == 1:
            commitishes = commitishes + [self.get_active_branch()]

        try:
            base = self.call_git_oneline(['merge-base'] + commitishes)
        except CommandError as exc:
            if exc.code == 1 and not (exc.stdout or exc.stderr):
                # No merge base was found (unrelated commits).
                return None
            if "fatal: Not a valid object name" in exc.stderr:
                return None
            raise

        return base

    def is_ancestor(self, reva, revb):
        """Is `reva` an ancestor of `revb`?

        Parameters
        ----------
        reva, revb : str
            Revisions.

        Returns
        -------
        bool
        """
        return self.call_git_success(
            ["merge-base", "--is-ancestor", reva, revb])

    def get_commit_date(self, branch=None, date='authored'):
        """Get the date stamp of the last commit (in a branch or head otherwise)

        Parameters
        ----------
        date: {'authored', 'committed'}
          Which date to return.  "authored" will be the date shown by "git show"
          and the one possibly specified via --date to `git commit`

        Returns
        -------
        int or None
          None if no commit
        """
        if date == 'committed':
            format = '%ct'
        elif date == 'authored':
            format = '%at'
        else:
            raise ValueError('unknow date type: {}'.format(date))
        d = self.format_commit(format, commitish=branch)
        return int(d) if d else None

    def get_active_branch(self):
        """Get the name of the active branch

        Returns
        -------
        str or None
          Returns None if there is no active branch, i.e. detached HEAD,
          and the branch name otherwise.
        """
        try:
            out, _ = self._git_custom_command(
                "", ["git", "symbolic-ref", "HEAD"],
                expect_fail=True)
        except CommandError as e:
            if 'HEAD is not a symbolic ref' in e.stderr:
                lgr.debug("detached HEAD in {0}".format(self))
                return None
            else:
                raise e
        return out.strip()[11:]  # strip refs/heads/

    def get_branches(self):
        """Get all branches of the repo.

        Returns
        -------
        [str]
            Names of all branches of this repository.
        """

        return [
            b['refname:strip=2']
            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/heads')
        ]

    def get_remote_branches(self):
        """Get all branches of all remotes of the repo.

        Returns
        -----------
        [str]
            Names of all remote branches.
        """
        # TODO: Reconsider melting with get_branches()

        # TODO: treat entries like this: origin/HEAD -> origin/master'
        # currently this is done in collection

        return [
            b['refname:strip=2']
            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/remotes')
        ]

    def get_remotes(self, with_urls_only=False):
        """Get known remotes of the repository

        Parameters
        ----------
        with_urls_only : bool, optional
          return only remotes which have urls

        Returns
        -------
        remotes : list of str
          List of names of the remotes
        """

        # Note: read directly from config and spare instantiation of gitpy.Repo
        # since we need this in AnnexRepo constructor. Furthermore gitpy does it
        # pretty much the same way and the use of a Repo instance seems to have
        # no reason other than a nice object oriented look.
        from datalad.utils import unique

        self.config.reload()
        remotes = unique([x[7:] for x in self.config.sections()
                          if x.startswith("remote.")])

        if with_urls_only:
            remotes = [
                r for r in remotes
                if self.config.get('remote.%s.url' % r)
            ]
        return remotes

    # TODO this is practically unused outside the tests, consider turning
    # into a test helper and trim from the API
    def get_files(self, branch=None):
        """Get a list of files in git.

        Lists the files in the (remote) branch.

        Parameters
        ----------
        branch: str
          Name of the branch to query. Default: active branch.

        Returns
        -------
        [str]
          list of files.
        """
        return [
            str(p.relative_to(self.pathobj))
            for p in self.get_content_info(
                paths=None, ref=branch, untracked='no', eval_file_type=False)
            ]

    @normalize_paths(match_return_type=False)
    def _git_custom_command(self, files, cmd_str,
                            log_stdout=True, log_stderr=True, log_online=False,
                            expect_stderr=True, cwd=None, env=None,
                            shell=None, expect_fail=False,
                            check_fake_dates=False,
                            index_file=None,
                            updates_tree=False):
        """Allows for calling arbitrary commands.

        Helper for developing purposes, i.e. to quickly implement git commands
        for proof of concept without the need to figure out, how this is done
        via GitPython.

        Parameters
        ----------
        files: list of files
        cmd_str: str or list
          arbitrary command str. `files` is appended to that string.
        updates_tree: bool
          whether or not command updates the working tree. If True, triggers
          necessary reevaluations like self.config.reload()

        Returns
        -------
        stdout, stderr
        """

        # ensure cmd_str becomes a well-formed list:
        if isinstance(cmd_str, str):
            cmd = split_cmdline(cmd_str)
        else:
            cmd = cmd_str[:]  # we will modify in-place

        assert(cmd[0] == 'git')
        cmd = cmd[:1] + self._GIT_COMMON_OPTIONS + cmd[1:]

        if check_fake_dates and self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        if index_file:
            env = (env if env is not None else os.environ).copy()
            env['GIT_INDEX_FILE'] = index_file

        # TODO?: wouldn't splitting interfer with above GIT_INDEX_FILE
        #  handling????
        try:
            out, err = run_gitcommand_on_file_list_chunks(
                self.cmd_call_wrapper.run,
                cmd,
                files,
                log_stderr=log_stderr,
                log_stdout=log_stdout,
                log_online=log_online,
                expect_stderr=expect_stderr,
                cwd=cwd,
                env=env,
                shell=shell,
                expect_fail=expect_fail)
        except CommandError as e:
            ignored = re.search(GitIgnoreError.pattern, e.stderr)
            if ignored:
                raise GitIgnoreError(cmd=e.cmd, msg=e.stderr,
                                     code=e.code, stdout=e.stdout,
                                     stderr=e.stderr,
                                     paths=ignored.groups()[0].splitlines())
            raise

        if updates_tree:
            lgr.debug("Reloading config due to supposed working tree update")
            self.config.reload()

        return out, err

    # Convenience wrappers for one-off git calls that don't require further
    # processing or error handling.

    def call_git(self, args, files=None,
                 expect_stderr=False, expect_fail=False):
        """Call git and return standard output.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        expect_fail : bool, optional
          A non-zero exit is expected and should not be elevated above the
          DEBUG level.

        Returns
        -------
        standard output (str)

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        """
        out, _ = self._git_custom_command(files, ["git"] + args,
                                          expect_stderr=expect_stderr,
                                          expect_fail=expect_fail,
                                          check_fake_dates=True)
        return out

    def call_git_items_(self, args, files=None, expect_stderr=False, sep=None):
        """Call git, splitting output on `sep`.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        sep : str, optional
          Split the output by `str.split(sep)` rather than `str.splitlines`.

        Returns
        -------
        Generator that yields output items.

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        """
        out, _ = self._git_custom_command(files, ["git"] + args,
                                          expect_stderr=expect_stderr,
                                          check_fake_dates=True)
        yield from (out.split(sep) if sep else out.splitlines())

    def call_git_oneline(self, args, files=None, expect_stderr=False):
        """Call git for a single line of output.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        sep : str, optional
          Split the output by `str.split(sep)` rather than `str.splitlines`.

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        AssertionError if there is more than one line of output.
        """
        lines = list(self.call_git_items_(args, files=files,
                                          expect_stderr=expect_stderr))
        if len(lines) > 1:
            raise AssertionError(
                "Expected {} to return single line, but it returned {}"
                .format(["git"] + args, lines))
        return lines[0]

    def call_git_success(self, args, files=None, expect_stderr=False):
        """Call git and return true if the call exit code of 0.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.

        Returns
        -------
        bool
        """
        try:
            self._git_custom_command(files, ["git"] + args,
                                     expect_fail=True,
                                     expect_stderr=expect_stderr,
                                     check_fake_dates=True)
        except CommandError:
            return False
        return True

    def add_remote(self, name, url, options=None):
        """Register remote pointing to a url
        """
        cmd = ['git', 'remote', 'add']
        if options:
            cmd += options
        cmd += [name, url]

        result = self._git_custom_command('', cmd)
        self.config.reload()
        return result

    def remove_remote(self, name):
        """Remove existing remote
        """

        # TODO: testing and error handling!
        from .exceptions import RemoteNotAvailableError
        try:
            out, err = self._git_custom_command(
                '', ['git', 'remote', 'remove', name])
        except CommandError as e:
            if 'fatal: No such remote' in e.stderr:
                raise RemoteNotAvailableError(name,
                                              cmd="git remote remove",
                                              msg="No such remote",
                                              stdout=e.stdout,
                                              stderr=e.stderr)
            else:
                raise e

        # TODO: config.reload necessary?
        self.config.reload()
        return

    def update_remote(self, name=None, verbose=False):
        """
        """
        options = ["-v"] if verbose else []
        name = [name] if name else []
        self._git_custom_command(
            '', ['git', 'remote'] + name + ['update'] + options,
            expect_stderr=True
        )

    def fetch(self, remote=None, refspec=None, all_=False, git_options=None,
              **kwargs):
        """Fetches changes from a remote (or all remotes).

        Parameters
        ----------
        remote : str, optional
          name of the remote to fetch from. If no remote is given and
          `all_` is not set, the tracking branch is fetched.
        refspec : str, optional
          refspec to fetch.
        all_ : bool, optional
          fetch all remotes (and all of their branches).
          Fails if `remote` was given.
        git_options : list, optional
          Additional command line options for git-fetch.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-fetch.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))
        return list(
            self.fetch_(
                remote=remote,
                refspec=refspec,
                all_=all_,
                git_options=git_options,
            )
        )

    @guard_BadName
    def fetch_(self, remote=None, refspec=None, all_=False, git_options=None):
        """Like `fetch`, but returns a generator"""
        yield from self._fetch_push_helper(
            base_cmd=['git', 'fetch', '--verbose', '--progress'],
            action='fetch',
            urlvars=('remote.{}.url', 'remote.{}.url'),
            protocol=GitProgress,
            info_cls=FetchInfo,
            info_from=1,
            add_remote=False,
            remote=remote,
            refspec=refspec,
            all_=all_,
            git_options=git_options)

    # XXX Consider removing this method. It is only used in `update()`,
    # where it could be easily replaced with fetch+merge
    def pull(self, remote=None, refspec=None, git_options=None, **kwargs):
        """Pulls changes from a remote.

        Parameters
        ----------
        remote : str, optional
          name of the remote to pull from. If no remote is given,
          the remote tracking branch is used.
        refspec : str, optional
          refspec to fetch.
        git_options : list, optional
          Additional command line options for git-pull.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-pull.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))

        cmd = ['git', 'pull', '--progress'] + git_options

        if remote is None:
            if refspec:
                # conflicts with using tracking branch or fetch all remotes
                # For now: Just fail.
                # TODO: May be check whether it fits to tracking branch
                raise ValueError(
                    "refspec specified without a remote. ({})".format(refspec))
            # No explicit remote to fetch.
            # => get tracking branch:
            tb_remote, refspec = self.get_tracking_branch()
            if tb_remote is not None:
                remote = tb_remote
            else:
                # No remote, no tracking branch
                # => fail
                raise ValueError("Neither a remote is specified to pull "
                                 "from nor a tracking branch is set up.")

        cmd.append(remote)
        if refspec:
            cmd += ensure_list(refspec)

        # best effort to enable SSH connection caching
        url = self.config.get('remote.{}.url'.format(remote), None)
        if url and is_ssh(url):
            ssh_manager.get_connection(url).open()
        WitlessRunner(
            cwd=self.path,
            env=GitRunner.get_git_environ_adjusted()).run(
                cmd,
                protocol=StdOutCaptureWithGitProgress,
        )

    def push(self, remote=None, refspec=None, all_remotes=False,
             all_=False, git_options=None, **kwargs):
        """Push changes to a remote (or all remotes).

        Parameters
        ----------
        remote : str, optional
          name of the remote to push to. If no remote is given and
          `all_` is not set, the tracking branch is pushed.
        refspec : str, optional
          refspec to push.
        all_ : bool, optional
          push to all remotes. Fails if `remote` was given.
        git_options : list, optional
          Additional command line options for git-push.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-push.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))
        if all_remotes:
            # be nice to the elderly
            all_ = True
        return list(
            self.push_(
                remote=remote,
                refspec=refspec,
                all_=all_,
                git_options=git_options,
            )
        )

    def push_(self, remote=None, refspec=None, all_=False, git_options=None):
        """Like `push`, but returns a generator"""
        yield from self._fetch_push_helper(
            base_cmd=['git', 'push', '--progress', '--porcelain'],
            action='push',
            urlvars=('remote.{}.pushurl', 'remote.{}.url'),
            protocol=StdOutCaptureWithGitProgress,
            info_cls=PushInfo,
            info_from=0,
            add_remote=True,
            remote=remote,
            refspec=refspec,
            all_=all_,
            git_options=git_options)

    def _fetch_push_helper(
            self,
            base_cmd,     # arg list
            action,       # label fetch|push
            urlvars,      # variables to query for URLs
            protocol,     # processor for output
            info_cls,     # Push|FetchInfo
            info_from,    # 0=stdout, 1=stderr
            add_remote,   # whether to add a 'remote' field to the info dict
            remote=None, refspec=None, all_=False, git_options=None):

        git_options = ensure_list(git_options)

        cmd = base_cmd + git_options

        if remote is None:
            if refspec:
                # conflicts with using tracking branch or push all remotes
                # For now: Just fail.
                # TODO: May be check whether it fits to tracking branch
                raise ValueError(
                    "refspec specified without a remote. ({})".format(refspec))
            if all_:
                remotes_to_process = self.get_remotes(with_urls_only=True)
            else:
                # No explicit remote to fetch.
                # => get tracking branch:
                tb_remote, refspec = self.get_tracking_branch()
                if tb_remote is not None:
                    remotes_to_process = [tb_remote]
                else:
                    # No remote, no tracking branch
                    # => fail
                    raise ValueError(
                        "Neither a remote is specified to {} "
                        "from nor a tracking branch is set up.".format(action))
        else:
            if all_:
                raise ValueError(
                    "Option 'all_' conflicts with specified remote "
                    "'{}'.".format(remote))
            remotes_to_process = [remote]

        if refspec:
            # prep for appending to cmd
            refspec = ensure_list(refspec)

        # no need for progress report, when there is just one remote
        log_remote_progress = len(remotes_to_process) > 1
        if log_remote_progress:
            pbar_id = '{}remotes-{}'.format(action, id(self))
            log_progress(
                lgr.info,
                pbar_id,
                'Start %sing remotes for %s', action, self,
                total=len(remotes_to_process),
                label=action.capitalize(),
                unit=' Remotes',
            )
        try:
            for remote in remotes_to_process:
                r_cmd = cmd + [remote]
                if refspec:
                    r_cmd += refspec

                if log_remote_progress:
                    log_progress(
                        lgr.info,
                        pbar_id,
                        '{}ing remote %s'.format(action.capitalize()),
                        remote,
                        update=1,
                        increment=True,
                    )
                # best effort to enable SSH connection caching
                url = self.config.get(
                    # make two attempts to get a URL
                    urlvars[0].format(remote),
                    self.config.get(
                        urlvars[1].format(remote),
                        None)
                )
                if url and is_ssh(url):
                    ssh_manager.get_connection(url).open()
                try:
                    out = WitlessRunner(
                        cwd=self.path,
                        env=GitRunner.get_git_environ_adjusted()).run(
                            r_cmd,
                            protocol=protocol,
                    )
                    output = out[info_from] or ''
                except CommandError as e:
                    # intercept some errors that we express as an error report
                    # in the info dicts
                    if re.match('error: failed to (push|fetch) some refs', e.stderr):
                        output = {1: e.stderr, 0: e.stdout}[info_from]
                        if output is None:
                            output = ''
                    else:
                        raise

                for line in output.splitlines():
                    try:
                        # push info doesn't identify a remote, add it here
                        pi = info_cls._from_line(line)
                        if add_remote:
                            pi['remote'] = remote
                        yield pi
                    except Exception:
                        # it is not progress and no push info
                        # don't hide it completely
                        lgr.debug('git-%s reported: %s', action, line)
        finally:
            if log_remote_progress:
                log_progress(
                    lgr.info,
                    pbar_id,
                    'Finished %sing remotes for %s', action, self,
                )

    def get_remote_url(self, name, push=False):
        """Get the url of a remote.

        Reads the configuration of remote `name` and returns its url or None,
        if there is no url configured.

        Parameters
        ----------
        name: str
          name of the remote
        push: bool
          if True, get the pushurl instead of the fetch url.
        """

        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')
        return self.config.get(var, None)

    def set_remote_url(self, name, url, push=False):
        """Set the URL a remote is pointing to

        Sets the URL of the remote `name`. Requires the remote to already exist.

        Parameters
        ----------
        name: str
          name of the remote
        url: str
        push: bool
          if True, set the push URL, otherwise the fetch URL
        """

        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')
        self.config.set(var, url, where='local', reload=True)

    def get_branch_commits_(self, branch=None, limit=None, stop=None):
        """Return commit hexshas for a branch

        Parameters
        ----------
        branch: str, optional
          If not provided, assumes current branch
        limit: None | 'left-only', optional
          Limit which commits to report.  If None -- all commits (merged or not),
          if 'left-only' -- only the commits from the left side of the tree upon
          merges
        stop: str, optional
          hexsha of the commit at which stop reporting (matched one is not
          reported either)

        Yields
        ------
        str
        """
        cmd = ['rev-list']
        if limit == 'left-only':
            cmd.append('--left-only')
        if not branch:
            branch = self.get_active_branch()
        cmd.append(branch)
        for r in self.call_git_items_(cmd):
            if stop and stop == r:
                return
            yield r

    def checkout(self, name, options=None):
        """
        """
        # TODO: May be check for the need of -b options herein?
        cmd = ['git', 'checkout']
        if options:
            cmd += options
        cmd += [str(name)]

        self._git_custom_command('', cmd, expect_stderr=True, updates_tree=True)

    # TODO: Before implementing annex merge, find usages and check for a needed
    # change to call super().merge
    def merge(self, name, options=None, msg=None, allow_unrelated=False, **kwargs):
        if options is None:
            options = []
        if msg:
            options = options + ["-m", msg]
        if allow_unrelated and external_versions['cmd:git'] >= '2.9':
            options += ['--allow-unrelated-histories']
        self._git_custom_command(
            '', ['git', 'merge'] + options + [name],
            check_fake_dates=True,
            **kwargs
        )

    def remove_branch(self, branch):
        self._git_custom_command(
            '', ['git', 'branch', '-D', branch]
        )

    def cherry_pick(self, commit):
        """Cherry pick `commit` to the current branch.

        Parameters
        ----------
        commit : str
            A single commit.
        """
        self._git_custom_command("", ["git", "cherry-pick", commit],
                                 check_fake_dates=True)

    @property
    def dirty(self):
        """Is the repository dirty?

        Note: This provides a quick answer when you simply want to know if
        there are any untracked changes or modifications in this repository or
        its submodules. For finer-grained control and more detailed reporting,
        use status() instead.
        """
        stdout, _ = self._git_custom_command(
            [],
            ["git", "status", "--porcelain",
             # Ensure the result isn't influenced by status.showUntrackedFiles.
             "--untracked-files=normal",
             # Ensure the result isn't influenced by diff.ignoreSubmodules.
             "--ignore-submodules=none"])
        return bool(stdout.strip())

    @property
    def untracked_files(self):
        """Legacy interface, do not use! Use the status() method instead.

        Despite its name, it also reports on untracked datasets, and
        yields their names with trailing path separators.
        """
        return [
            '{}{}'.format(
                str(p.relative_to(self.pathobj)),
                os.sep if props['type'] != 'file' else ''
            )
            for p, props in self.status(
                    untracked='all', eval_submodule_state='no').items()
            if props.get('state', None) == 'untracked'
        ]

    def gc(self, allow_background=False, auto=False):
        """Perform house keeping (garbage collection, repacking)"""
        cmd_options = ['git']
        if not allow_background:
            cmd_options += ['-c', 'gc.autodetach=0']
        cmd_options += ['gc', '--aggressive']
        if auto:
            cmd_options += ['--auto']
        self._git_custom_command('', cmd_options)

    def _parse_gitmodules(self):
        # TODO read .gitconfig from Git blob?
        gitmodules = self.pathobj / '.gitmodules'
        if not gitmodules.exists():
            return {}
        # pull out file content
        out, err = self._git_custom_command(
            '',
            ['git', 'config', '-z', '-l', '--file', '.gitmodules'])
        # abuse our config parser
        db, _ = _parse_gitconfig_dump(out, {}, None, True, cwd=self.path)
        mods = {}
        for k, v in db.items():
            if not k.startswith('submodule.'):
                # we don't know what this is
                lgr.warning("Skip unrecognized .gitmodule specification: %s=%s", k, v)
                continue
            k_l = k.split('.')
            # module name is everything after 'submodule.' that is not the variable
            # name
            mod_name = '.'.join(k_l[1:-1])
            mod = mods.get(mod_name, {})
            # variable name is the last 'dot-free' segment in the key
            mod[k_l[-1]] = v
            mods[mod_name] = mod

        out = {}
        # bring into traditional shape
        for name, props in mods.items():
            if 'path' not in props:
                lgr.warning("Failed to get '%s.path', skipping this submodule", name)
                continue
            modprops = {'gitmodule_{}'.format(k): v
                        for k, v in props.items()
                        if not (k.startswith('__') or k == 'path')}
            modpath = self.pathobj / PurePosixPath(props['path'])
            modprops['gitmodule_name'] = name
            out[modpath] = modprops
        return out

    def get_submodules_(self, paths=None):
        """Yield submodules in this repository.

        Parameters
        ----------
        paths : list(pathlib.PurePath), optional
            Restrict submodules to those under `paths`.

        Returns
        -------
        A generator that yields a dictionary with information for each
        submodule.
        """
        if not (self.pathobj / ".gitmodules").exists():
            return

        modinfo = self._parse_gitmodules()
        for path, props in self.get_content_info(
                paths=paths,
                ref=None,
                untracked='no',
                eval_file_type=False).items():
            if props.get('type', None) != 'dataset':
                continue
            props["path"] = path
            props.update(modinfo.get(path, {}))
            yield props

    def get_submodules(self, sorted_=True, paths=None, compat=True):
        """Return list of submodules.

        Parameters
        ----------
        sorted_ : bool, optional
            Sort submodules by path name.
        paths : list(pathlib.PurePath), optional
            Restrict submodules to those under `paths`.
        compat : bool, optional
            If true, return a namedtuple that incompletely mimics the
            attributes of GitPython's Submodule object in hope of backwards
            compatibility with previous callers. Note that this form should be
            considered temporary and callers should be updated; this flag will
            be removed in a future release.

        Returns
        -------
        List of submodule namedtuples if `compat` is true or otherwise a list
        of dictionaries as returned by `get_submodules_`.
        """
        xs = self.get_submodules_(paths=paths)
        if compat:
            warnings.warn("The attribute-based return value of get_submodules() "
                          "exists for compatibility purposes and will be removed "
                          "in an upcoming release",
                          DeprecationWarning)
            xs = (Submodule(name=p["gitmodule_name"],
                            path=str(p["path"].relative_to(self.pathobj)),
                            url=p["gitmodule_url"])
                  for p in xs)

        if sorted_:
            if compat:
                def key(x):
                    return x.path
            else:
                def key(x):
                    return x["path"]
            xs = sorted(xs, key=key)
        return list(xs)

    def add_submodule(self, path, name=None, url=None, branch=None):
        """Add a new submodule to the repository.

        This will alter the index as well as the .gitmodules file, but will not
        create a new commit.  If the submodule already exists, no matter if the
        configuration differs from the one provided, the existing submodule
        is considered as already added and no further action is performed.

        Parameters
        ----------
        path : str
          repository-relative path at which the submodule should be located, and
          which will be created as required during the repository initialization.
        name : str or None
          name/identifier for the submodule. If `None`, the `path` will be used
          as name.
        url : str or None
          git-clone compatible URL. If `None`, the repository is assumed to
          exist, and the url of the first remote is taken instead. This is
          useful if you want to make an existing repository a submodule of
          another one.
        branch : str or None
          name of branch to be checked out in the submodule. The given branch
          must exist in the remote repository, and will be checked out locally
          as a tracking branch. If `None`, remote HEAD will be checked out.
        """
        if name is None:
            name = Path(path).as_posix()
        # XXX the following should do it, but GitPython will refuse to add a submodule
        # unless you specify a URL that is configured as one of its remotes, or you
        # specify no URL, but the repo has at least one remote.
        # this is stupid, as for us it is valid to not have any remote, because we can
        # still obtain the submodule from a future publication location, based on the
        # parent
        # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch)
        # going git native instead
        cmd = ['submodule', 'add', '--name', name]
        if branch is not None:
            cmd += ['-b', branch]
        if url is None:
            # repo must already exist locally
            subm = GitRepo(op.join(self.path, path), create=False, init=False)
            # check that it has a commit, and refuse
            # to operate on it otherwise, or we would get a bastard
            # submodule that cripples git operations
            if not subm.get_hexsha():
                raise InvalidGitRepositoryError(
                    'cannot add subdataset {} with no commits'.format(subm))
            # make an attempt to configure a submodule source URL based on the
            # discovered remote configuration
            remote, branch = subm.get_tracking_branch()
            url = subm.get_remote_url(remote) if remote else None

        if url is None:
            # had no luck with a remote URL
            if not isabs(path):
                # need to recode into a relative path "URL" in POSIX
                # style, even on windows
                url = posixpath.join(curdir, posix_relpath(path))
            else:
                url = path
        cmd += [url, Path(path).as_posix()]
        self.call_git(cmd)
        # record dataset ID if possible for comprehesive metadata on
        # dataset components within the dataset itself
        subm_id = GitRepo(op.join(self.path, path)).config.get(
            'datalad.dataset.id', None)
        if subm_id:
            self.call_git(
                ['config', '--file', '.gitmodules', '--replace-all',
                 'submodule.{}.datalad-id'.format(name), subm_id])
        # ensure supported setup
        _fixup_submodule_dotgit_setup(self, path)
        # TODO: return value

    def deinit_submodule(self, path, **kwargs):
        """Deinit a submodule

        Parameters
        ----------
        path: str
            path to the submodule; relative to `self.path`
        kwargs:
            see `__init__`
        """

        self._git_custom_command(path,
                                 ['git', 'submodule', 'deinit'] +
                                 to_options(**kwargs))
        # TODO: return value

    def update_submodule(self, path, mode='checkout', init=False):
        """Update a registered submodule.

        This will make the submodule match what the superproject expects by
        cloning missing submodules and updating the working tree of the
        submodules. The "updating" can be done in several ways depending
        on the value of submodule.<name>.update configuration variable, or
        the `mode` argument.

        Parameters
        ----------
        path : str
          Identifies which submodule to operate on by it's repository-relative
          path.
        mode : {checkout, rebase, merge}
          Update procedure to perform. 'checkout': the commit recorded in the
          superproject will be checked out in the submodule on a detached HEAD;
          'rebase': the current branch of the submodule will be rebased onto
          the commit recorded in the superproject; 'merge': the commit recorded
          in the superproject will be merged into the current branch in the
          submodule.
        init : bool
          If True, initialize all submodules for which "git submodule init" has
          not been called so far before updating.
          Primarily provided for internal purposes and should not be used directly
          since would result in not so annex-friendly .git symlinks/references
          instead of full featured .git/ directories in the submodules
        """
        if GitRepo.is_valid_repo(self.pathobj / path):
            subrepo = GitRepo(self.pathobj / path, create=False)
            subbranch = subrepo.get_active_branch() if subrepo else None
            subbranch_hexsha = subrepo.get_hexsha(subbranch) if subrepo else None
        else:
            subrepo = None
            subbranch = None
            subbranch_hexsha = None

        cmd = ['git', 'submodule', 'update', '--%s' % mode]
        if init:
            cmd.append('--init')
            subgitpath = opj(self.path, path, '.git')
            if not exists(subgitpath):
                # TODO:  wouldn't with --init we get all those symlink'ed .git/?
                # At least let's warn
                lgr.warning(
                    "Do not use update_submodule with init=True to avoid git creating "
                    "symlinked .git/ directories in submodules"
                )
            #  yoh: I thought I saw one recently but thought it was some kind of
            #  an artifact from running submodule update --init manually at
            #  some point, but looking at this code now I worry that it was not
        self._git_custom_command(path, cmd)

        if not init:
            return

        # track branch originally cloned, only if we had a valid repo at the start
        updated_subbranch = subrepo.get_active_branch() if subrepo else None
        if subbranch and not updated_subbranch:
            # got into 'detached' mode
            # trace if current state is a predecessor of the branch_hexsha
            lgr.debug(
                "Detected detached HEAD after updating submodule %s which was "
                "in %s branch before", self.path, subbranch)
            detached_hexsha = subrepo.get_hexsha()
            if subrepo.get_merge_base(
                    [subbranch_hexsha, detached_hexsha]) == detached_hexsha:
                # TODO: config option?
                # in all likely event it is of the same branch since
                # it is an ancestor -- so we could update that original branch
                # to point to the state desired by the submodule, and update
                # HEAD to point to that location
                lgr.info(
                    "Submodule HEAD got detached. Resetting branch %s to point "
                    "to %s. Original location was %s",
                    subbranch, detached_hexsha[:8], subbranch_hexsha[:8]
                )
                branch_ref = 'refs/heads/%s' % subbranch
                subrepo.update_ref(branch_ref, detached_hexsha)
                assert(subrepo.get_hexsha(subbranch) == detached_hexsha)
                subrepo.update_ref('HEAD', branch_ref, symbolic=True)
                assert(subrepo.get_active_branch() == subbranch)
            else:
                lgr.warning(
                    "%s has a detached HEAD since cloned branch %s has another common ancestor with %s",
                    subrepo.path, subbranch, detached_hexsha[:8]
                )
        # TODO: return value

    def update_ref(self, ref, value, symbolic=False):
        """Update the object name stored in a ref "safely".

        Just a shim for `git update-ref` call if not symbolic, and
        `git symbolic-ref` if symbolic

        Parameters
        ----------
        ref : str
          Reference, such as `ref/heads/BRANCHNAME` or HEAD.
        value : str
          Value to update to, e.g. hexsha of a commit when updating for a
          branch ref, or branch ref if updating HEAD
        symbolic : None
          To instruct if ref is symbolic, e.g. should be used in case of
          ref=HEAD
        """
        self._git_custom_command(
            '',
            ['git', 'symbolic-ref' if symbolic else 'update-ref', ref, value]
        )

    def tag(self, tag, message=None, commit=None, options=None):
        """Tag a commit

        Parameters
        ----------
        tag : str
          Custom tag label. Must be a valid tag name.
        message : str, optional
          If provided, adds ['-m', <message>] to the list of `git tag`
          arguments.
        commit : str, optional
          If provided, will be appended as last argument to the `git tag` call,
          and can be used to identify the commit that shall be tagged, if
          not HEAD.
        options : list, optional
          Additional command options, inserted prior a potential `commit`
          argument.
        """
        # TODO: call in save.py complains about extensive logging. When does it
        # happen in what way? Figure out, whether to just silence it or raise or
        # whatever else.
        args = ['tag']
        if message:
            args += ['-m', message]
        if options is not None:
            args.extend(options)
        args.append(tag)
        if commit:
            args.append(commit)
        self.call_git(args)

    def get_tags(self, output=None):
        """Get list of tags

        Parameters
        ----------
        output : str, optional
          If given, limit the return value to a list of values matching that
          particular key of the tag properties.

        Returns
        -------
        list
          Each item is a dictionary with information on a tag. At present
          this includes 'hexsha', and 'name', where the latter is the string
          label of the tag, and the former the hexsha of the object the tag
          is attached to. The list is sorted by the creator date (committer
          date for lightweight tags and tagger date for annotated tags), with
          the most recent commit being the last element.
        """
        tags = [
            dict(
                name=t['refname:strip=2'],
                hexsha=t['object'] if t['object'] else t['objectname'],
            )
            for t in self.for_each_ref_(
                fields=['refname:strip=2', 'objectname', 'object'],
                pattern='refs/tags',
                sort='creatordate')
        ]
        if output:
            return [t[output] for t in tags]
        else:
            return tags

    def describe(self, commitish=None, **kwargs):
        """ Quick and dirty implementation to call git-describe

        Parameters
        ----------
        kwargs:
            transformed to cmdline options for git-describe;
            see __init__ for description of the transformation
        """
        # TODO: be more precise what failure to expect when and raise actual
        # errors
        cmd = ['git', 'describe'] + to_options(**kwargs)
        if commitish is not None:
            cmd.append(commitish)
        try:
            describe, outerr = self._git_custom_command(
                [],
                cmd,
                expect_fail=True)
            return describe.strip()
        # TODO: WTF "catch everything"?
        except:
            return None

    def get_tracking_branch(self, branch=None, remote_only=False):
        """Get the tracking branch for `branch` if there is any.

        Parameters
        ----------
        branch: str
            local branch to look up. If none is given, active branch is used.
        remote_only : bool
            Don't return a value if the upstream remote is set to "." (meaning
            this repository).

        Returns
        -------
        tuple
            (remote or None, refspec or None) of the tracking branch
        """
        if branch is None:
            branch = self.get_active_branch()
            if branch is None:
                return None, None

        track_remote = self.config.get('branch.{0}.remote'.format(branch), None)
        if remote_only and track_remote == ".":
            return None, None
        track_branch = self.config.get('branch.{0}.merge'.format(branch), None)
        return track_remote, track_branch

    @property
    def count_objects(self):
        """return dictionary with count, size(in KiB) information of git objects
        """

        count_cmd = ['git', 'count-objects', '-v']
        count_str, err = self._git_custom_command('', count_cmd)
        count = {key: int(value)
                 for key, value in [item.split(': ')
                                    for item in count_str.split('\n')
                                    if len(item.split(': ')) == 2]}
        return count

    def get_git_attributes(self):
        """Query gitattributes which apply to top level directory

        It is a thin compatibility/shortcut wrapper around more versatile
        get_gitattributes which operates on a list of paths and returns
        a dictionary per each path

        Returns
        -------
        dict:
          a dictionary with attribute name and value items relevant for the
          top ('.') directory of the repository, and thus most likely the
          default ones (if not overwritten with more rules) for all files within
          repo.
        """
        return self.get_gitattributes('.')['.']

    def get_gitattributes(self, path, index_only=False):
        """Query gitattributes for one or more paths

        Parameters
        ----------
        path: path or list
          Path(s) to query. Paths may be relative or absolute.
        index_only: bool
          Flag whether to consider only gitattribute setting that are reflected
          in the repository index, not just in the work tree content.

        Returns
        -------
        dict:
          Each key is a queried path (always relative to the repostiory root),
          each value is a dictionary with attribute
          name and value items. Attribute values are either True or False,
          for set and unset attributes, or are the literal attribute value.
        """
        path = ensure_list(path)
        cmd = ["git", "check-attr", "-z", "--all"]
        if index_only:
            cmd.append('--cached')
        stdout, stderr = self._git_custom_command(path, cmd)
        # make sure we have one entry for each query path to
        # simplify work with the result
        attributes = {_normalize_path(self.path, p): {} for p in path}
        attr = []
        for item in stdout.split('\0'):
            attr.append(item)
            if len(attr) < 3:
                continue
            # we have a full record
            p, name, value = attr
            attrs = attributes[p]
            attrs[name] = \
                True if value == 'set' else False if value == 'unset' else value
            # done, reset item
            attr = []
        return attributes

    def set_gitattributes(self, attrs, attrfile='.gitattributes', mode='a'):
        """Set gitattributes

        By default appends additional lines to `attrfile`. Note, that later
        lines in `attrfile` overrule earlier ones, which may or may not be
        what you want. Set `mode` to 'w' to replace the entire file by
        what you provided in `attrs`.

        Parameters
        ----------
        attrs : list
          Each item is a 2-tuple, where the first element is a path pattern,
          and the second element is a dictionary with attribute key/value
          pairs. The attribute dictionary must use the same semantics as those
          returned by `get_gitattributes()`. Path patterns can use absolute paths,
          in which case they will be normalized relative to the directory
          that contains the target .gitattributes file (see `attrfile`).
        attrfile: path
          Path relative to the repository root of the .gitattributes file the
          attributes shall be set in.
        mode: str
          'a' to append .gitattributes, 'w' to replace it
        """

        git_attributes_file = op.join(self.path, attrfile)
        attrdir = op.dirname(git_attributes_file)
        if not op.exists(attrdir):
            os.makedirs(attrdir)
        with open(git_attributes_file, mode) as f:
            for pattern, attr in sorted(attrs, key=lambda x: x[0]):
                # normalize the pattern relative to the target .gitattributes file
                npath = _normalize_path(
                    op.join(self.path, op.dirname(attrfile)), pattern)
                # paths in gitattributes always have to be POSIX
                npath = Path(npath).as_posix()
                attrline = u''
                if npath.count(' '):
                    # quote patterns with spaces
                    attrline += u'"{}"'.format(npath.replace('"', '\\"'))
                else:
                    attrline += npath
                for a in sorted(attr):
                    val = attr[a]
                    if val is True:
                        attrline += ' {}'.format(a)
                    elif val is False:
                        attrline += ' -{}'.format(a)
                    else:
                        attrline += ' {}={}'.format(a, val)
                f.write('\n{}'.format(attrline))

    def get_content_info(self, paths=None, ref=None, untracked='all',
                         eval_file_type=True):
        """Get identifier and type information from repository content.

        This is simplified front-end for `git ls-files/tree`.

        Both commands differ in their behavior when queried about subdataset
        paths. ls-files will not report anything, ls-tree will report on the
        subdataset record. This function uniformly follows the behavior of
        ls-tree (report on the respective subdataset mount).

        Parameters
        ----------
        paths : list(pathlib.PurePath)
          Specific paths, relative to the resolved repository root, to query
          info for. Paths must be normed to match the reporting done by Git,
          i.e. no parent dir components (ala "some/../this").
          If none are given, info is reported for all content.
        ref : gitref or None
          If given, content information is retrieved for this Git reference
          (via ls-tree), otherwise content information is produced for the
          present work tree (via ls-files). With a given reference, the
          reported content properties also contain a 'bytesize' record,
          stating the size of a file in bytes.
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported when no `ref` was given:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_file_type : bool
          If True, inspect file type of untracked files, and report annex
          symlink pointers as type 'file'. This convenience comes with a
          cost; disable to get faster performance if this information
          is not needed.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'

            Note that the reported type will not always match the type of
            content committed to Git, rather it will reflect the nature
            of the content minus platform/mode-specifics. For example,
            a symlink to a locked annexed file on Unix will have a type
            'file', reported, while a symlink to a file in Git or directory
            will be of type 'symlink'.

          `gitshasum`
            SHASUM of the item as tracked by Git, or None, if not
            tracked. This could be different from the SHASUM of the file
            in the worktree, if it was modified.

        Raises
        ------
        ValueError
          In case of an invalid Git reference (e.g. 'HEAD' in an empty
          repository)
        """
        lgr.debug('%s.get_content_info(...)', self)
        # TODO limit by file type to replace code in subdatasets command
        info = OrderedDict()

        if paths:
            # path matching will happen against what Git reports
            # and Git always reports POSIX paths
            # any incoming path has to be relative already, so we can simply
            # convert unconditionally
            paths = [ut.PurePosixPath(p) for p in paths]

        path_strs = list(map(str, paths)) if paths else None

        # this will not work in direct mode, but everything else should be
        # just fine
        if not ref:
            # make sure no operations are pending before we figure things
            # out in the worktree
            self.precommit()

            # --exclude-standard will make sure to honor and standard way
            # git can be instructed to ignore content, and will prevent
            # crap from contaminating untracked file reports
            cmd = ['git', 'ls-files',
                   '--stage', '-z', '-d', '-m', '--exclude-standard']
            # untracked report mode, using labels from `git diff` option style
            if untracked == 'all':
                cmd.append('-o')
            elif untracked == 'normal':
                cmd += ['-o', '--directory', '--no-empty-directory']
            elif untracked == 'no':
                pass
            else:
                raise ValueError(
                    'unknown value for `untracked`: {}'.format(untracked))
            props_re = re.compile(
                r'(?P<type>[0-9]+) (?P<sha>.*) (.*)\t(?P<fname>.*)$')

            if path_strs:
                # we need to get their within repo elements since ls-tree
                # for paths within submodules returns nothing!
                # see https://public-inbox.org/git/20190703193305.GF21553@hopa.kiewit.dartmouth.edu/T/#u
                submodules = [s.path for s in self.get_submodules()]
                path_strs = get_parent_paths(path_strs, submodules)
        else:
            cmd = ['git', 'ls-tree', ref, '-z', '-r', '--full-tree', '-l']
            props_re = re.compile(
                r'(?P<type>[0-9]+) ([a-z]*) (?P<sha>[^ ]*) [\s]*(?P<size>[0-9-]+)\t(?P<fname>.*)$')

        lgr.debug('Query repo: %s', cmd)
        try:
            stdout, stderr = self._git_custom_command(
                path_strs,
                cmd,
                log_stderr=True,
                log_stdout=True,
                # not sure why exactly, but log_online has to be false!
                log_online=False,
                expect_stderr=False,
                shell=False,
                # we don't want it to scream on stdout
                expect_fail=True)
        except CommandError as exc:
            if "fatal: Not a valid object name" in str(exc):
                raise InvalidGitReferenceError(ref)
            raise
        lgr.debug('Done query repo: %s', cmd)

        if not eval_file_type:
            _get_link_target = None
        elif ref:
            def _read_symlink_target_from_catfile(lines):
                # it is always the second line, all checks done upfront
                header = lines.readline()
                if header.rstrip().endswith('missing'):
                    # something we do not know about, should not happen
                    # in real use, but guard against to avoid stalling
                    return ''
                return lines.readline().rstrip()

            _get_link_target = BatchedCommand(
                ['git', 'cat-file', '--batch'],
                path=self.path,
                output_proc=_read_symlink_target_from_catfile,
            )
        else:
            def try_readlink(path):
                try:
                    return os.readlink(path)
                except OSError:
                    # readlink will fail if the symlink reported by ls-files is
                    # not in the working tree (it could be removed or
                    # unlocked). Fall back to a slower method.
                    return op.realpath(path)

            _get_link_target = try_readlink

        try:
            self._get_content_info_line_helper(
                ref,
                info,
                stdout.split('\0'),
                props_re,
                _get_link_target)
        finally:
            if ref and _get_link_target:
                # cancel batch process
                _get_link_target.close()

        lgr.debug('Done %s.get_content_info(...)', self)
        return info

    def _get_content_info_line_helper(self, ref, info, lines,
                                      props_re, get_link_target):
        """Internal helper of get_content_info() to parse Git output"""
        mode_type_map = {
            '100644': 'file',
            '100755': 'file',
            '120000': 'symlink',
            '160000': 'dataset',
        }
        for line in lines:
            if not line:
                continue
            inf = {}
            props = props_re.match(line)
            if not props:
                # not known to Git, but Git always reports POSIX
                path = ut.PurePosixPath(line)
                inf['gitshasum'] = None
            else:
                # again Git reports always in POSIX
                path = ut.PurePosixPath(props.group('fname'))

            # revisit the file props after this path has not been rejected
            if props:
                inf['gitshasum'] = props.group('sha')
                inf['type'] = mode_type_map.get(
                    props.group('type'), props.group('type'))
                if get_link_target and inf['type'] == 'symlink' and \
                        ((ref is None and '.git/annex/objects' in \
                          ut.Path(
                            get_link_target(str(self.pathobj / path))
                          ).as_posix()) or \
                         (ref and \
                          '.git/annex/objects' in get_link_target(
                              u'{}:{}'.format(
                                  ref, str(path))))
                        ):
                    # report annex symlink pointers as file, their
                    # symlink-nature is a technicality that is dependent
                    # on the particular mode annex is in
                    inf['type'] = 'file'

                if ref and inf['type'] == 'file':
                    inf['bytesize'] = int(props.group('size'))

            # join item path with repo path to get a universally useful
            # path representation with auto-conversion and tons of other
            # stuff
            path = self.pathobj.joinpath(path)
            if 'type' not in inf:
                # be nice and assign types for untracked content
                inf['type'] = 'symlink' if path.is_symlink() \
                    else 'directory' if path.is_dir() else 'file'
            info[path] = inf

    def status(self, paths=None, untracked='all', eval_submodule_state='full'):
        """Simplified `git status` equivalent.

        Parameters
        ----------
        paths : list or None
          If given, limits the query to the specified paths. To query all
          paths specify `None`, not an empty list. If a query path points
          into a subdataset, a report is made on the subdataset record
          within the queried dataset only (no recursion).
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_submodule_state : {'full', 'commit', 'no'}
          If 'full' (the default), the state of a submodule is evaluated by
          considering all modifications, with the treatment of untracked files
          determined by `untracked`. If 'commit', the modification check is
          restricted to comparing the submodule's HEAD commit to the one
          recorded in the superdataset. If 'no', the state of the subdataset is
          not evaluated.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'
          `state`
            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.
        """
        lgr.debug('Query status of %r for %s paths',
                  self, len(paths) if paths else 'all')
        return self.diffstatus(
            fr='HEAD' if self.get_hexsha() else None,
            to=None,
            paths=paths,
            untracked=untracked,
            eval_submodule_state=eval_submodule_state)

    def diff(self, fr, to, paths=None, untracked='all',
             eval_submodule_state='full'):
        """Like status(), but reports changes between to arbitrary revisions

        Parameters
        ----------
        fr : str or None
          Revision specification (anything that Git understands). Passing
          `None` considers anything in the target state as new.
        to : str or None
          Revision specification (anything that Git understands), or None
          to compare to the state of the work tree.
        paths : list or None
          If given, limits the query to the specified paths. To query all
          paths specify `None`, not an empty list.
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported when `to` is None:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_submodule_state : {'full', 'commit', 'no'}
          If 'full' (the default), the state of a submodule is evaluated by
          considering all modifications, with the treatment of untracked files
          determined by `untracked`. If 'commit', the modification check is
          restricted to comparing the submodule's HEAD commit to the one
          recorded in the superdataset. If 'no', the state of the subdataset is
          not evaluated.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'
          `state`
            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.
        """
        return {k: v for k, v in self.diffstatus(
            fr=fr, to=to, paths=paths,
            untracked=untracked,
            eval_submodule_state=eval_submodule_state).items()
            if v.get('state', None) != 'clean'}

    def diffstatus(self, fr, to, paths=None, untracked='all',
                   eval_submodule_state='full', eval_file_type=True,
                   _cache=None):
        """Like diff(), but reports the status of 'clean' content too"""
        return self._diffstatus(
            fr, to, paths, untracked, eval_submodule_state, eval_file_type,
            _cache)

    def _diffstatus(self, fr, to, paths, untracked, eval_state,
                    eval_file_type, _cache):
        """Just like diffstatus(), but supports an additional evaluation
        state 'global'. If given, it will return a single 'modified'
        (vs. 'clean') state label for the entire repository, as soon as
        it can."""
        def _get_cache_key(label, paths, ref, untracked=None):
            return self.path, label, tuple(paths) if paths else None, \
                ref, untracked

        if _cache is None:
            _cache = {}

        if paths:
            # at this point we must normalize paths to the form that
            # Git would report them, to easy matching later on
            paths = [ut.Path(p) for p in paths]
            paths = [
                p.relative_to(self.pathobj) if p.is_absolute() else p
                for p in paths
            ]

        # TODO report more info from get_content_info() calls in return
        # value, those are cheap and possibly useful to a consumer
        # we need (at most) three calls to git
        if to is None:
            # everything we know about the worktree, including os.stat
            # for each file
            key = _get_cache_key('ci', paths, None, untracked)
            if key in _cache:
                to_state = _cache[key]
            else:
                to_state = self.get_content_info(
                    paths=paths, ref=None, untracked=untracked,
                    eval_file_type=eval_file_type)
                _cache[key] = to_state
            # we want Git to tell us what it considers modified and avoid
            # reimplementing logic ourselves
            key = _get_cache_key('mod', paths, None)
            if key in _cache:
                modified = _cache[key]
            else:
                modified = set(
                    self.pathobj.joinpath(ut.PurePosixPath(p))
                    for p in self._git_custom_command(
                        # low-level code cannot handle pathobjs
                        [str(p) for p in paths] if paths else None,
                        ['git', 'ls-files', '-z', '-m'])[0].split('\0')
                    if p)
                _cache[key] = modified
        else:
            key = _get_cache_key('ci', paths, to)
            if key in _cache:
                to_state = _cache[key]
            else:
                to_state = self.get_content_info(
                    paths=paths, ref=to, eval_file_type=eval_file_type)
                _cache[key] = to_state
            # we do not need worktree modification detection in this case
            modified = None
        # origin state
        key = _get_cache_key('ci', paths, fr)
        if key in _cache:
            from_state = _cache[key]
        else:
            if fr:
                from_state = self.get_content_info(
                    paths=paths, ref=fr, eval_file_type=eval_file_type)
            else:
                # no ref means from nothing
                from_state = {}
            _cache[key] = from_state

        status = OrderedDict()
        for f, to_state_r in to_state.items():
            props = None
            if f not in from_state:
                # this is new, or rather not known to the previous state
                props = dict(
                    state='added' if to_state_r['gitshasum'] else 'untracked',
                )
                if 'type' in to_state_r:
                    props['type'] = to_state_r['type']
            elif to_state_r['gitshasum'] == from_state[f]['gitshasum'] and \
                    (modified is None or f not in modified):
                if to_state_r['type'] != 'dataset':
                    # no change in git record, and no change on disk
                    props = dict(
                        # at this point we know that the reported object ids
                        # for this file are identical in the to and from
                        # records.  If to is None, we're comparing to the
                        # working tree and a deleted file will still have an
                        # identical id, so we need to check whether the file is
                        # gone before declaring it clean. This working tree
                        # check is irrelevant and wrong if to is a ref.
                        state='clean' if to is not None or (f.exists() or \
                              f.is_symlink()) else 'deleted',
                        type=to_state_r['type'],
                    )
                else:
                    # a dataset
                    props = dict(type=to_state_r['type'])
                    if to is not None:
                        # we can only be confident without looking
                        # at the worktree, if we compare to a recorded
                        # state
                        props['state'] = 'clean'
                    else:
                        # report the shasum that we know, for further
                        # wrangling of subdatasets below
                        props['gitshasum'] = to_state_r['gitshasum']
                        props['prev_gitshasum'] = from_state[f]['gitshasum']
            else:
                # change in git record, or on disk
                props = dict(
                    # TODO we could have a new file that is already staged
                    # but had subsequent modifications done to it that are
                    # unstaged. Such file would presently show up as 'added'
                    # ATM I think this is OK, but worth stating...
                    state='modified' if f.exists() or \
                    f.is_symlink() else 'deleted',
                    # TODO record before and after state for diff-like use
                    # cases
                    type=to_state_r['type'],
                )
            state = props.get('state', None)
            if eval_state == 'global' and \
                    state not in ('clean', None):
                # any modification means globally 'modified'
                return 'modified'
            if state in ('clean', 'added', 'modified'):
                props['gitshasum'] = to_state_r['gitshasum']
                if 'bytesize' in to_state_r:
                    # if we got this cheap, report it
                    props['bytesize'] = to_state_r['bytesize']
                elif props['state'] == 'clean' and 'bytesize' in from_state[f]:
                    # no change, we can take this old size info
                    props['bytesize'] = from_state[f]['bytesize']
            if state in ('clean', 'modified', 'deleted'):
                props['prev_gitshasum'] = from_state[f]['gitshasum']
            status[f] = props

        for f, from_state_r in from_state.items():
            if f not in to_state:
                # we new this, but now it is gone and Git is not complaining
                # about it being missing -> properly deleted and deletion
                # stages
                status[f] = dict(
                    state='deleted',
                    type=from_state_r['type'],
                    # report the shasum to distinguish from a plainly vanished
                    # file
                    gitshasum=from_state_r['gitshasum'],
                )
                if eval_state == 'global':
                    return 'modified'

        if to is not None or eval_state == 'no':
            # if we have `to` we are specifically comparing against
            # a recorded state, and this function only attempts
            # to label the state of a subdataset, not investigate
            # specifically what the changes in subdatasets are
            # this is done by a high-level command like rev-diff
            # so the comparison within this repo and the present
            # `state` label are all we need, and they are done already
            if eval_state == 'global':
                return 'clean'
            else:
                return status

        # loop over all subdatasets and look for additional modifications
        for f, st in status.items():
            f = str(f)
            if 'state' in st or not st['type'] == 'dataset':
                # no business here
                continue
            if not GitRepo.is_valid_repo(f):
                # submodule is not present, no chance for a conflict
                st['state'] = 'clean'
                continue
            # we have to recurse into the dataset and get its status
            subrepo = GitRepo(f)
            subrepo_commit = subrepo.get_hexsha()
            st['gitshasum'] = subrepo_commit
            # subdataset records must be labeled clean up to this point
            # test if current commit in subdataset deviates from what is
            # recorded in the dataset
            st['state'] = 'modified' \
                if st['prev_gitshasum'] != subrepo_commit \
                else 'clean'
            if eval_state == 'global' and st['state'] == 'modified':
                return 'modified'
            if eval_state == 'commit':
                continue
            # the recorded commit did not change, so we need to make
            # a more expensive traversal
            st['state'] = subrepo._diffstatus(
                # we can use 'HEAD' because we know that the commit
                # did not change. using 'HEAD' will facilitate
                # caching the result
                fr='HEAD',
                to=None,
                paths=None,
                untracked=untracked,
                eval_state='global',
                eval_file_type=False,
                _cache=_cache) if st['state'] == 'clean' else 'modified'
            if eval_state == 'global' and st['state'] == 'modified':
                return 'modified'

        if eval_state == 'global':
            return 'clean'
        else:
            return status

    def _save_pre(self, paths, _status, **kwargs):
        # helper to get an actionable status report
        if paths is not None and not paths and not _status:
            return
        if _status is None:
            if 'untracked' not in kwargs:
                kwargs['untracked'] = 'normal'
            status = self.status(
                paths=paths,
                **{k: kwargs[k] for k in kwargs
                   if k in ('untracked', 'eval_submodule_state')})
        else:
            # we want to be able to add items down the line
            # make sure to detach from prev. owner
            status = _status.copy()
        status = OrderedDict(
            (k, v) for k, v in status.items()
            if v.get('state', None) != 'clean'
        )
        return status

    def get_staged_paths(self):
        """Returns a list of any stage repository path(s)

        This is a rather fast call, as it will not depend on what is going on
        in the worktree.
        """
        try:
            stdout, stderr = self._git_custom_command(
                None,
                ['git', 'diff', '--name-only', '--staged'],
                cwd=self.path,
                log_stderr=True,
                log_stdout=True,
                log_online=False,
                expect_stderr=False,
                expect_fail=True)
        except CommandError as e:
            lgr.debug(exc_str(e))
            stdout = ''
        return [f for f in stdout.split('\n') if f]

    def _save_post(self, message, status, partial_commit):
        # helper to commit changes reported in status

        # TODO remove pathobj stringification when commit() can
        # handle it
        to_commit = [str(f.relative_to(self.pathobj))
                     for f, props in status.items()] \
                    if partial_commit else None
        if not partial_commit or to_commit:
            # we directly call GitRepo.commit() to avoid a whole slew
            # if direct-mode safeguards and workarounds in the AnnexRepo
            # implementation (which also run an additional dry-run commit
            GitRepo.commit(
                self,
                files=to_commit,
                msg=message,
                options=None,
                # do not raise on empty commit
                # it could be that the `add` in this save-cycle has already
                # brought back a 'modified' file into a clean state
                careless=True,
            )

    def save(self, message=None, paths=None, _status=None, **kwargs):
        """Save dataset content.

        Parameters
        ----------
        message : str or None
          A message to accompany the changeset in the log. If None,
          a default message is used.
        paths : list or None
          Any content with path matching any of the paths given in this
          list will be saved. Matching will be performed against the
          dataset status (GitRepo.status()), or a custom status provided
          via `_status`. If no paths are provided, ALL non-clean paths
          present in the repo status or `_status` will be saved.
        _status : dict or None
          If None, Repo.status() will be queried for the given `ds`. If
          a dict is given, its content will be used as a constraint.
          For example, to save only modified content, but no untracked
          content, set `paths` to None and provide a `_status` that has
          no entries for untracked content.
        **kwargs :
          Additional arguments that are passed to underlying Repo methods.
          Supported:

          - git : bool (passed to Repo.add()
          - eval_submodule_state : {'full', 'commit', 'no'}
            passed to Repo.status()
          - untracked : {'no', 'normal', 'all'} - passed to Repo.satus()
        """
        return list(
            self.save_(
                message=message,
                paths=paths,
                _status=_status,
                **kwargs
            )
        )

    def save_(self, message=None, paths=None, _status=None, **kwargs):
        """Like `save()` but working as a generator."""
        from datalad.interface.results import get_status_dict

        status = self._save_pre(paths, _status, **kwargs)
        if not status:
            # all clean, nothing todo
            lgr.debug('Nothing to save in %r, exiting early', self)
            return

        # three things are to be done:
        # - remove (deleted if not already staged)
        # - add (modified/untracked)
        # - commit (with all paths that have been touched, to bypass
        #   potential pre-staged bits)

        need_partial_commit = True if self.get_staged_paths() else False

        # remove first, because removal of a subds would cause a
        # modification of .gitmodules to be added to the todo list
        to_remove = [
            # TODO remove pathobj stringification when delete() can
            # handle it
            str(f.relative_to(self.pathobj))
            for f, props in status.items()
            if props.get('state', None) == 'deleted' and
            # staged deletions have a gitshasum reported for them
            # those should not be processed as git rm will error
            # due to them being properly gone already
            not props.get('gitshasum', None)]
        vanished_subds = any(
            props.get('type', None) == 'dataset' and
            props.get('state', None) == 'deleted'
            for f, props in status.items())
        if to_remove:
            for r in self.remove(
                    to_remove,
                    # we would always see individual files
                    recursive=False):
                # TODO normalize result
                yield get_status_dict(
                    action='delete',
                    refds=self.pathobj,
                    # TODO make remove() report the type
                    # for now it claims to report on files only
                    type='file',
                    path=(self.pathobj / ut.PurePosixPath(r)),
                    # make remove() report on failures too
                    status='ok',
                    logger=lgr)

        # TODO this additonal query should not be, base on status as given
        # if anyhow possible, however, when paths are given, status may
        # not contain all required information. In case of path=None AND
        # _status=None, we should be able to avoid this, because
        # status should have the full info already
        # looks for contained repositories
        added_submodule = False
        untracked_dirs = [f.relative_to(self.pathobj)
                          for f, props in status.items()
                          if props.get('state', None) == 'untracked' and
                          props.get('type', None) == 'directory']
        to_add_submodules = []
        if untracked_dirs:
            to_add_submodules = [sm for sm, sm_props in
                self.get_content_info(
                    untracked_dirs,
                    ref=None,
                    # request exhaustive list, so that everything that is
                    # still reported as a directory must be its own repository
                    untracked='all').items()
                if sm_props.get('type', None) == 'directory']
            to_add_submodules = _prune_deeper_repos(to_add_submodules)
            for cand_sm in to_add_submodules:
                branch = self.get_active_branch()
                adjusted_match = ADJUSTED_BRANCH_EXPR.match(
                    branch if branch else '')
                try:
                    self.add_submodule(
                        str(cand_sm.relative_to(self.pathobj)),
                        url=None,
                        name=None,
                        branch=adjusted_match.group('name') if adjusted_match
                        else branch
                    )
                except (CommandError, InvalidGitRepositoryError) as e:
                    yield get_status_dict(
                        action='add_submodule',
                        ds=self,
                        path=self.pathobj / ut.PurePosixPath(cand_sm),
                        status='error',
                        message=e.stderr if hasattr(e, 'stderr')
                        else ('not a Git repository: %s', exc_str(e)),
                        logger=lgr)
                    continue
                # This mirrors the result structure yielded for
                # to_stage_submodules below.
                yield get_status_dict(
                    action='add',
                    refds=self.pathobj,
                    type='file',
                    key=None,
                    path=self.pathobj / ut.PurePosixPath(cand_sm),
                    status='ok',
                    logger=lgr)
                added_submodule = True
        if not need_partial_commit:
            # without a partial commit an AnnexRepo would ignore any submodule
            # path in its add helper, hence `git add` them explicitly
            to_stage_submodules = {
                str(f.relative_to(self.pathobj)): props
                for f, props in status.items()
                if props.get('state', None) in ('modified', 'untracked')
                and props.get('type', None) == 'dataset'}
            if to_stage_submodules:
                lgr.debug(
                    '%i submodule path(s) to stage in %r %s',
                    len(to_stage_submodules), self,
                    to_stage_submodules
                    if len(to_stage_submodules) < 10 else '')
                for r in GitRepo._save_add(
                        self,
                        to_stage_submodules,
                        git_opts=None):
                    yield r

        if added_submodule or vanished_subds:
            # need to include .gitmodules in what needs saving
            status[self.pathobj.joinpath('.gitmodules')] = dict(
                type='file', state='modified')
            if hasattr(self, 'annexstatus') and not kwargs.get('git', False):
                # we cannot simply hook into the coming add-call
                # as this would go to annex, so make a dedicted git-add
                # call to ensure .gitmodules is not annexed
                # in any normal DataLad dataset .gitattributes will
                # prevent this, but in a plain repo it won't
                # https://github.com/datalad/datalad/issues/3306
                for r in GitRepo._save_add(
                        self,
                        {op.join(self.path, '.gitmodules'): None}):
                    yield r
        to_add = {
            # TODO remove pathobj stringification when add() can
            # handle it
            str(f.relative_to(self.pathobj)): props
            for f, props in status.items()
            if (props.get('state', None) in ('modified', 'untracked') and
                f not in to_add_submodules)}
        if to_add:
            lgr.debug(
                '%i path(s) to add to %s %s',
                len(to_add), self, to_add if len(to_add) < 10 else '')
            for r in self._save_add(
                    to_add,
                    git_opts=None,
                    **{k: kwargs[k] for k in kwargs
                       if k in (('git',) if hasattr(self, 'annexstatus')
                                else tuple())}):
                yield r

        self._save_post(message, status, need_partial_commit)
        # TODO yield result for commit, prev helper checked hexsha pre
        # and post...

    def _save_add(self, files, git_opts=None):
        """Simple helper to add files in save()"""
        from datalad.interface.results import get_status_dict
        try:
            # without --verbose git 2.9.3  add does not return anything
            add_out = self._git_custom_command(
                list(files.keys()),
                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']
            )
            # get all the entries
            for r in self._process_git_get_output(*add_out):
                yield get_status_dict(
                    action=r.get('command', 'add'),
                    refds=self.pathobj,
                    type='file',
                    path=(self.pathobj / ut.PurePosixPath(r['file']))
                    if 'file' in r else None,
                    status='ok' if r.get('success', None) else 'error',
                    key=r.get('key', None),
                    # while there is no git-annex underneath here, we
                    # tend to fake its behavior, so we can also support
                    # this type of messaging
                    message='\n'.join(r['error-messages'])
                    if 'error-messages' in r else None,
                    logger=lgr)
        except OSError as e:
            lgr.error("add: %s" % e)
            raise


# TODO
# remove submodule: nope, this is just deinit_submodule + remove
# status?


def _fixup_submodule_dotgit_setup(ds, relativepath):
    """Implementation of our current of .git in a subdataset

    Each subdataset/module has its own .git directory where a standalone
    repository would have it. No gitdir files, no symlinks.
    """
    # move .git to superrepo's .git/modules, remove .git, create
    # .git-file
    path = opj(ds.path, relativepath)
    subds_dotgit = opj(path, ".git")

    repo = GitRepo(path, create=False)
    if repo.dot_git.parent == repo.pathobj:
        # this is what we want
        return

    # first we want to remove any conflicting worktree setup
    # done by git to find the checkout at the mountpoint of the
    # submodule, if we keep that, any git command will fail
    # after we move .git
    # Ben: Shouldn't we re-setup a possible worktree afterwards?
    repo.config.unset('core.worktree', where='local')
    # what we have here is some kind of reference, remove and
    # replace by the target
    os.remove(subds_dotgit)
    # make absolute
    src_dotgit = str(repo.dot_git)
    # move .git
    from os import rename, listdir, rmdir
    ensure_dir(subds_dotgit)
    for dot_git_entry in listdir(src_dotgit):
        rename(opj(src_dotgit, dot_git_entry),
               opj(subds_dotgit, dot_git_entry))
    assert not listdir(src_dotgit)
    rmdir(src_dotgit)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** `to_options` function rearranges command-line options, causing test mismatches

**Description:**
After updating the `to_options` function by importing code from GitPython, the order of generated command-line options has changed. This alteration leads to mismatches in tests that expect options in a specific sequence.

**Example Code:**
```python
def cmd_func(options=None):
    return ['git', 'annex', 'my_cmd'] + to_options(m='bla', force=True)

output = cmd_func()
print(output)
# Expected: ['git', 'annex', 'my_cmd', '--force', '-m', 'bla']
# Actual: ['git', 'annex', 'my_cmd', '-m', 'bla', '--force']
```

**Expected Behavior:**
The `to_options` function should generate command-line options in a consistent order, such as `--force` followed by `-m`, to match the expected command sequences in tests.

**Actual Behavior:**
The options are reordered, with `-m` appearing before `--force`, causing tests to fail due to the unexpected sequence.

[/ISSUE]
{
    "test_GitRepo_files_decorator": "PASSED",
    "test_to_options": "PASSED",
    "test_to_options_from_gitpython": "PASSED",
    "test_guard_BadName": "PASSED",
    "test_GitRepo_instance_from_clone": "ERROR",
    "test_GitRepo_instance_from_not_existing": "ERROR",
    "test_GitRepo_equals": "ERROR",
    "test_GitRepo_add": "ERROR",
    "test_GitRepo_get_indexed_files": "ERROR",
    "test_normalize_path": "ERROR",
    "test_GitRepo_remote_add": "ERROR",
    "test_GitRepo_remote_remove": "ERROR",
    "test_GitRepo_get_remote_url": "ERROR",
    "test_GitRepo_pull": "ERROR",
    "test_GitRepo_fetch": "ERROR",
    "test_GitRepo_ssh_fetch": "ERROR",
    "test_GitRepo_ssh_pull": "ERROR",
    "test_GitRepo_ssh_push": "ERROR",
    "test_GitRepo_push_n_checkout": "ERROR",
    "test_GitRepo_remote_update": "ERROR",
    "test_GitRepo_get_files": "ERROR",
    "test_GitRepo_get_toppath": "ERROR",
    "test_GitRepo_get_merge_base": "ERROR",
    "test_GitRepo_git_get_branch_commits_": "ERROR",
    "test_get_tracking_branch": "ERROR",
    "test_GitRepo_add_submodule": "ERROR",
    "test_GitRepo_count_objects": "ERROR",
    "test_GitRepo_gitpy_injection": "ERROR",
    "test_GitRepo_flyweight": "ERROR",
    "test_GitRepo_flyweight_monitoring_inode": "ERROR",
    "test_GitRepo_invalid_path": "FAILED",
    "test_GitRepo_instance_from_existing": "FAILED",
    "test_GitRepo_init_options": "FAILED",
    "test_init_fail_under_known_subdir": "FAILED",
    "test_GitRepo_remove": "FAILED",
    "test_GitRepo_commit": "FAILED",
    "test_GitRepo_dirty": "FAILED",
    "test_submodule_deinit": "FAILED",
    "test_get_submodules_parent_on_unborn_branch": "FAILED",
    "test_optimized_cloning": "FAILED",
    "test_GitRepo_gitignore": "FAILED",
    "test_GitRepo_set_remote_url": "FAILED",
    "test_gitattributes": "FAILED",
    "test_get_hexsha_tag": "FAILED",
    "test_get_tags": "FAILED",
    "test_get_commit_date": "FAILED",
    "test_fake_dates": "FAILED",
    "test_custom_runner_protocol": "FAILED",
    "test_duecredit": "FAILED",
    "test_GitRepo_get_revisions": "FAILED",
    "test_gitrepo_add_to_git_with_annex_v7": "FAILED",
    "test_gitrepo_call_git_methods": "FAILED"
}
diff --git a/datalad/support/gitrepo.py b/datalad/support/gitrepo.py
index 0764e6c3f..098260c77 100644
--- a/datalad/support/gitrepo.py
+++ b/datalad/support/gitrepo.py
@@ -137,9 +137,16 @@ default_git_odbt = gitpy.GitCmdObjectDB
 # log Exceptions from git commands.
 
 
-def to_options(**kwargs):
+def to_options(split_single_char_options=True, **kwargs):
     """Transform keyword arguments into a list of cmdline options
 
+    Imported from GitPython.
+
+    Original copyright:
+        Copyright (C) 2008, 2009 Michael Trier and contributors
+    Original license:
+        BSD 3-Clause "New" or "Revised" License
+
     Parameters
     ----------
     split_single_char_options: bool
@@ -150,9 +157,34 @@ def to_options(**kwargs):
     -------
     list
     """
-    # TODO: borrow_docs!
-
-    return gitpy.Git().transform_kwargs(**kwargs)
+    def dashify(string):
+        return string.replace('_', '-')
+
+    def transform_kwarg(name, value, split_single_char_options):
+        if len(name) == 1:
+            if value is True:
+                return ["-%s" % name]
+            elif value not in (False, None):
+                if split_single_char_options:
+                    return ["-%s" % name, "%s" % value]
+                else:
+                    return ["-%s%s" % (name, value)]
+        else:
+            if value is True:
+                return ["--%s" % dashify(name)]
+            elif value is not False and value is not None:
+                return ["--%s=%s" % (dashify(name), value)]
+        return []
+
+    args = []
+    kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0]))
+    for k, v in kwargs.items():
+        if isinstance(v, (list, tuple)):
+            for value in v:
+                args += transform_kwarg(k, value, split_single_char_options)
+        else:
+            args += transform_kwarg(k, v, split_single_char_options)
+    return args
 
 
 def _normalize_path(base_dir, path):

{"datalad/support/gitrepo.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to Git via GitPython\n\nFor further information on GitPython see http://gitpython.readthedocs.org/\n\n\"\"\"\n\nimport re\nimport time\nimport os\nimport os.path as op\nimport warnings\nfrom locale import getpreferredencoding\n\n\nfrom itertools import chain\nimport logging\nfrom collections import (\n    OrderedDict,\n    namedtuple\n)\nfrom os import linesep\nfrom os.path import (\n    join as opj,\n    exists,\n    normpath,\n    isabs,\n    commonprefix,\n    relpath,\n    realpath,\n    dirname,\n    basename,\n    curdir,\n    pardir,\n    sep\n)\n\nimport posixpath\nfrom functools import wraps\nfrom weakref import WeakValueDictionary\n\nimport git as gitpy\nfrom gitdb.exc import BadName\nfrom git.exc import (\n    GitCommandError,\n    NoSuchPathError,\n    InvalidGitRepositoryError\n)\nfrom datalad.log import log_progress\nfrom datalad.support.due import due, Doi\n\nfrom datalad import ssh_manager\nfrom datalad.cmd import (\n    WitlessRunner,\n    WitlessProtocol,\n    GitRunner,\n    BatchedCommand,\n    run_gitcommand_on_file_list_chunks,\n    StdOutErrCapture,\n)\nfrom datalad.config import (\n    ConfigManager,\n    _parse_gitconfig_dump\n)\n\nfrom datalad.consts import (\n    ADJUSTED_BRANCH_EXPR,\n)\nfrom datalad.dochelpers import exc_str\nimport datalad.utils as ut\nfrom datalad.utils import (\n    Path,\n    PurePosixPath,\n    ensure_list,\n    optional_args,\n    on_windows,\n    getpwd,\n    posix_relpath,\n    ensure_dir,\n    generate_file_chunks,\n    ensure_unicode,\n    split_cmdline,\n)\n\n# imports from same module:\nfrom .external_versions import external_versions\nfrom .exceptions import (\n    CommandError,\n    DeprecatedError,\n    FileNotInRepositoryError,\n    GitIgnoreError,\n    InvalidGitReferenceError,\n    MissingBranchError,\n    OutdatedExternalDependencyWarning,\n    PathKnownToRepositoryError,\n)\nfrom .network import (\n    RI,\n    PathRI,\n    is_ssh\n)\nfrom .path import get_parent_paths\nfrom .repo import (\n    PathBasedFlyweight,\n    RepoInterface\n)\n\n#from git.objects.blob import Blob\n\n# shortcuts\n_curdirsep = curdir + sep\n_pardirsep = pardir + sep\n\n\nlgr = logging.getLogger('datalad.gitrepo')\n_lgr_level = lgr.getEffectiveLevel()\nif _lgr_level <= 2:\n    from ..log import LoggerHelper\n    # Let's also enable gitpy etc debugging\n    gitpy_lgr = LoggerHelper(logtarget=\"git\").get_initialized_logger()\n    gitpy_lgr.setLevel(_lgr_level)\n    gitpy_lgr.propagate = True\n\n# Override default GitPython's DB backend to talk directly to git so it doesn't\n# interfere with possible operations performed by gc/repack\ndefault_git_odbt = gitpy.GitCmdObjectDB\n\n# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used.\n# Didn't work as expected on a first try. Probably there is a neatier way to\n# log Exceptions from git commands.\n\n\ndef to_options(**kwargs):\n    \"\"\"Transform keyword arguments into a list of cmdline options\n\n    Parameters\n    ----------\n    split_single_char_options: bool\n\n    kwargs:\n\n    Returns\n    -------\n    list\n    \"\"\"\n    # TODO: borrow_docs!\n\n    return gitpy.Git().transform_kwargs(**kwargs)\n\n\ndef _normalize_path(base_dir, path):\n    \"\"\"Helper to check paths passed to methods of this class.\n\n    Checks whether `path` is beneath `base_dir` and normalizes it.\n    Additionally paths are converted into relative paths with respect to\n    `base_dir`, considering PWD in case of relative paths. This\n    is intended to be used in repository classes, which means that\n    `base_dir` usually will be the repository's base directory.\n\n    Parameters\n    ----------\n    base_dir: str\n        directory to serve as base to normalized, relative paths\n    path: str\n        path to be normalized\n\n    Returns\n    -------\n    str:\n        path, that is a relative path with respect to `base_dir`\n    \"\"\"\n    if not path:\n        return path\n\n    base_dir = realpath(base_dir)  # realpath OK\n    # path = normpath(path)\n    # Note: disabled normpath, because it may break paths containing symlinks;\n    # But we don't want to realpath relative paths, in case cwd isn't the\n    # correct base.\n\n    if isabs(path):\n        # path might already be a symlink pointing to annex etc,\n        # so realpath only its directory, to get \"inline\" with\n        # realpath(base_dir) above\n        path = opj(realpath(dirname(path)), basename(path))  # realpath OK\n    # Executive decision was made to not do this kind of magic!\n    #\n    # elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:\n    #     # If we are inside repository, rebuilt relative paths.\n    #     path = opj(realpath(getpwd()), path)\n    #\n    # BUT with relative curdir/pardir start it would assume relative to curdir\n    #\n    elif path.startswith(_curdirsep) or path.startswith(_pardirsep):\n        path = normpath(opj(realpath(getpwd()), path))  # realpath OK\n    else:\n        # We were called from outside the repo. Therefore relative paths\n        # are interpreted as being relative to self.path already.\n        return path\n\n    if commonprefix([path, base_dir]) != base_dir:\n        raise FileNotInRepositoryError(msg=\"Path outside repository: %s\"\n                                           % path, filename=path)\n\n    return relpath(path, start=base_dir)\n\n\n@optional_args\ndef normalize_path(func):\n    \"\"\"Decorator to provide unified path conversion for a single file\n\n    Unlike normalize_paths, intended to be used for functions dealing with a\n    single filename at a time\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, file_, *args, **kwargs):\n        file_new = _normalize_path(self.path, file_)\n        return func(self, file_new, *args, **kwargs)\n\n    return newfunc\n\n\n@optional_args\ndef normalize_paths(func, match_return_type=True, map_filenames_back=False,\n                    serialize=False):\n    \"\"\"Decorator to provide unified path conversions.\n\n    Note\n    ----\n    This is intended to be used within the repository classes and therefore\n    returns a class method!\n\n    The decorated function is expected to take a path or a list of paths at\n    first positional argument (after 'self'). Additionally the class `func`\n    is a member of, is expected to have an attribute 'path'.\n\n    Accepts either a list of paths or a single path in a str. Passes a list\n    to decorated function either way, but would return based on the value of\n    match_return_type and possibly input argument.\n\n    If a call to the wrapped function includes normalize_path and it is False\n    no normalization happens for that function call (used for calls to wrapped\n    functions within wrapped functions, while possible CWD is within a\n    repository)\n\n    Parameters\n    ----------\n    match_return_type : bool, optional\n      If True, and a single string was passed in, it would return the first\n      element of the output (after verifying that it is a list of length 1).\n      It makes easier to work with single files input.\n    map_filenames_back : bool, optional\n      If True and returned value is a dictionary, it assumes to carry entries\n      one per file, and then filenames are mapped back to as provided from the\n      normalized (from the root of the repo) paths\n    serialize : bool, optional\n      Loop through files giving only a single one to the function one at a time.\n      This allows to simplify implementation and interface to annex commands\n      which do not take multiple args in the same call (e.g. checkpresentkey)\n    \"\"\"\n\n    @wraps(func)\n    def newfunc(self, files, *args, **kwargs):\n\n        normalize = _normalize_path if kwargs.pop('normalize_paths', True) \\\n            else lambda rpath, filepath: filepath\n\n        if files:\n            if isinstance(files, str) or not files:\n                files_new = [normalize(self.path, files)]\n                single_file = True\n            elif isinstance(files, list):\n                files_new = [normalize(self.path, path) for path in files]\n                single_file = False\n            else:\n                raise ValueError(\"_files_decorator: Don't know how to handle \"\n                                 \"instance of %s.\" % type(files))\n        else:\n            single_file = None\n            files_new = []\n\n        if map_filenames_back:\n            def remap_filenames(out):\n                \"\"\"Helper to map files back to non-normalized paths\"\"\"\n                if isinstance(out, dict):\n                    assert(len(out) == len(files_new))\n                    files_ = [files] if single_file else files\n                    mapped = out.__class__()\n                    for fin, fout in zip(files_, files_new):\n                        mapped[fin] = out[fout]\n                    return mapped\n                else:\n                    return out\n        else:\n            remap_filenames = lambda x: x\n\n        if serialize:  # and not single_file:\n            result = [\n                func(self, f, *args, **kwargs)\n                for f in files_new\n            ]\n        else:\n            result = func(self, files_new, *args, **kwargs)\n\n        if single_file is None:\n            # no files were provided, nothing we can do really\n            return result\n        elif (result is None) or not match_return_type or not single_file:\n            # If function doesn't return anything or no denormalization\n            # was requested or it was not a single file\n            return remap_filenames(result)\n        elif single_file:\n            if len(result) != 1:\n                # Magic doesn't apply\n                return remap_filenames(result)\n            elif isinstance(result, (list, tuple)):\n                return result[0]\n            elif isinstance(result, dict) and tuple(result)[0] == files_new[0]:\n                # assume that returned dictionary has files as keys.\n                return tuple(result.values())[0]\n            else:\n                # no magic can apply\n                return remap_filenames(result)\n        else:\n            return RuntimeError(\"should have not got here... check logic\")\n\n    return newfunc\n\n\ndef _remove_empty_items(list_):\n    \"\"\"Remove empty entries from list\n\n    This is needed, since some functions of GitPython may convert\n    an empty entry to '.', when used with a list of paths.\n\n    Parameter:\n    ----------\n    list_: list of str\n\n    Returns\n    -------\n    list of str\n    \"\"\"\n    if not isinstance(list_, list):\n        lgr.warning(\n            \"_remove_empty_items() called with non-list type: %s\" % type(list_))\n        return list_\n    return [file_ for file_ in list_ if file_]\n\n\nif \"2.24.0\" <= external_versions[\"cmd:git\"] < \"2.25.0\":\n    # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing\n    # into untracked submodules when multiple pathspecs are given, returning\n    # repositories that are deeper than the first level. This helper filters\n    # these deeper levels out so that save_() doesn't fail trying to add them.\n    #\n    # This regression fixed with upstream's 072a231016 (2019-12-10).\n    def _prune_deeper_repos(repos):\n        firstlevel_repos = []\n        prev = None\n        for repo in sorted(repos):\n            if not (prev and str(repo).startswith(prev)):\n                prev = str(repo)\n                firstlevel_repos.append(repo)\n        return firstlevel_repos\nelse:\n    def _prune_deeper_repos(repos):\n        return repos\n\n\ndef Repo(*args, **kwargs):\n    \"\"\"Factory method around gitpy.Repo to consistently initiate with different\n    backend\n    \"\"\"\n    # TODO: This probably doesn't work as intended (or at least not as\n    #       consistently as intended). gitpy.Repo could be instantiated by\n    #       classmethods Repo.init or Repo.clone_from. In these cases 'odbt'\n    #       would be needed as a parameter to these methods instead of the\n    #       constructor.\n    if 'odbt' not in kwargs:\n        kwargs['odbt'] = default_git_odbt\n    return gitpy.Repo(*args, **kwargs)\n\n\ndef guard_BadName(func):\n    \"\"\"A helper to guard against BadName exception\n\n    Workaround for\n    https://github.com/gitpython-developers/GitPython/issues/768\n    also see https://github.com/datalad/datalad/issues/2550\n    Let's try to precommit (to flush anything flushable) and do\n    it again\n    \"\"\"\n\n    @wraps(func)\n    def wrapped(repo, *args, **kwargs):\n        try:\n            return func(repo, *args, **kwargs)\n        except BadName:\n            repo.precommit()\n            return func(repo, *args, **kwargs)\n\n    return wrapped\n\n\nclass GitProgress(WitlessProtocol):\n    \"\"\"Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    # inform super-class to capture stderr\n    proc_err = True\n\n    _num_op_codes = 10\n    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \\\n        [1 << x for x in range(_num_op_codes)]\n    STAGE_MASK = BEGIN | END\n    OP_MASK = ~STAGE_MASK\n\n    DONE_TOKEN = 'done.'\n    TOKEN_SEPARATOR = ', '\n\n    _known_ops = {\n        COUNTING: (\"Counting\", \"Objects\"),\n        ENUMERATING: (\"Enumerating\", \"Objects\"),\n        COMPRESSING: (\"Compressing\", \"Objects\"),\n        WRITING: (\"Writing\", \"Objects\"),\n        RECEIVING: (\"Receiving\", \"Objects\"),\n        RESOLVING: (\"Resolving\", \"Deltas\"),\n        FINDING_SOURCES: (\"Finding\", \"Sources\"),\n        CHECKING_OUT: (\"Check out\", \"Things\"),\n    }\n\n    __slots__ = ('_seen_ops', '_pbars', '_encoding')\n\n    re_op_absolute = re.compile(r\"(remote: )?([\\w\\s]+):\\s+()(\\d+)()(.*)\")\n    re_op_relative = re.compile(r\"(remote: )?([\\w\\s]+):\\s+(\\d+)% \\((\\d+)/(\\d+)\\)(.*)\")\n\n    def __init__(self, *args):\n        super().__init__(*args)\n        self._encoding = getpreferredencoding(do_setlocale=False)\n        self._unprocessed = None\n\n    def connection_made(self, transport):\n        super().connection_made(transport)\n        self._seen_ops = []\n        self._pbars = set()\n\n    def process_exited(self):\n        # take down any progress bars that were not closed orderly\n        for pbar_id in self._pbars:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished',\n            )\n        super().process_exited()\n\n    def pipe_data_received(self, fd, byts):\n        # progress reports only come from stderr\n        if fd != 2:\n            # let the base class decide what to do with it\n            super().pipe_data_received(fd, byts)\n            return\n        for line in byts.splitlines(keepends=True):\n            # put any unprocessed content back in front\n            line = self._unprocessed + line if self._unprocessed else line\n            self._unprocessed = None\n            if not self._parse_progress_line(line):\n                # anything that doesn't look like a progress report\n                # is retained and returned\n                # in case of partial progress lines, this can lead to\n                # leakage of progress info into the output, but\n                # it is better to enable better (maybe more expensive)\n                # subsequent filtering than hidding lines with\n                # unknown, potentially important info\n                lgr.debug('Non-progress stderr: %s', line)\n                if line.endswith((b'\\r', b'\\n')):\n                    # complete non-progress line, pass on\n                    super().pipe_data_received(fd, line)\n                else:\n                    # an incomplete line, maybe the next batch completes\n                    # it to become a recognizable progress report\n                    self._unprocessed = line\n\n    def _parse_progress_line(self, line):\n        \"\"\"Process a single line\n\n        Parameters\n        ----------\n        line : bytes\n\n        Returns\n        -------\n        bool\n          Flag whether the line was recognized as a Git progress report.\n        \"\"\"\n        # handle\n        # Counting objects: 4, done.\n        # Compressing objects:  50% (1/2)\n        # Compressing objects: 100% (2/2)\n        # Compressing objects: 100% (2/2), done.\n        line = line.decode(self._encoding) if isinstance(line, bytes) else line\n        if line.startswith(('warning:', 'error:', 'fatal:')):\n            return False\n\n        # find escape characters and cut them away - regex will not work with\n        # them as they are non-ascii. As git might expect a tty, it will send them\n        last_valid_index = None\n        for i, c in enumerate(reversed(line)):\n            if ord(c) < 32:\n                # its a slice index\n                last_valid_index = -i - 1\n            # END character was non-ascii\n        # END for each character in line\n        if last_valid_index is not None:\n            line = line[:last_valid_index]\n        # END cut away invalid part\n        line = line.rstrip()\n\n        cur_count, max_count = None, None\n        match = self.re_op_relative.match(line)\n        if match is None:\n            match = self.re_op_absolute.match(line)\n\n        if not match:\n            return False\n        # END could not get match\n\n        op_code = 0\n        _remote, op_name, _percent, cur_count, max_count, message = match.groups()\n\n        # get operation id\n        if op_name == \"Counting objects\":\n            op_code |= self.COUNTING\n        elif op_name == \"Compressing objects\":\n            op_code |= self.COMPRESSING\n        elif op_name == \"Writing objects\":\n            op_code |= self.WRITING\n        elif op_name == 'Receiving objects':\n            op_code |= self.RECEIVING\n        elif op_name == 'Resolving deltas':\n            op_code |= self.RESOLVING\n        elif op_name == 'Finding sources':\n            op_code |= self.FINDING_SOURCES\n        elif op_name == 'Checking out files':\n            op_code |= self.CHECKING_OUT\n        elif op_name == 'Enumerating objects':\n            op_code |= self.ENUMERATING\n        else:\n            # Note: On windows it can happen that partial lines are sent\n            # Hence we get something like \"CompreReceiving objects\", which is\n            # a blend of \"Compressing objects\" and \"Receiving objects\".\n            # This can't really be prevented.\n            lgr.debug(\n                'Output line matched a progress report of an unknown type: %s',\n                line)\n            # TODO investigate if there is any chance that we might swallow\n            # important info -- until them do not flag this line\n            # as progress\n            return False\n        # END handle op code\n\n        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)\n\n        op_props = self._known_ops[op_code]\n\n        # figure out stage\n        if op_code not in self._seen_ops:\n            self._seen_ops.append(op_code)\n            op_code |= self.BEGIN\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n                label=op_props[0],\n                unit=' {}'.format(op_props[1]),\n                total=float(max_count) if max_count else None,\n            )\n            self._pbars.add(pbar_id)\n        # END begin opcode\n\n        if message is None:\n            message = ''\n        # END message handling\n\n        done_progress = False\n        message = message.strip()\n        if message.endswith(self.DONE_TOKEN):\n            op_code |= self.END\n            message = message[:-len(self.DONE_TOKEN)]\n            done_progress = True\n        # END end message handling\n        message = message.strip(self.TOKEN_SEPARATOR)\n\n        if cur_count and max_count:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                message,\n                update=float(cur_count),\n            )\n\n        if done_progress:\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Finished {} {}'.format(\n                    op_props[0].lower(),\n                    op_props[1].lower(),\n                ),\n            )\n            self._pbars.discard(pbar_id)\n        return True\n\n\nclass StdOutCaptureWithGitProgress(GitProgress):\n    proc_out = True\n\n\nclass FetchInfo(dict):\n    \"\"\"\n    dict that carries results of a fetch operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n\n    NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \\\n        FAST_FORWARD, ERROR = [1 << x for x in range(8)]\n\n    _re_fetch_result = re.compile(r'^\\s*(.) (\\[?[\\w\\s\\.$@]+\\]?)\\s+(.+) [-> ]+ ([^\\s]+)(    \\(.*\\)?$)?')\n\n    _flag_map = {\n        '!': ERROR,\n        '+': FORCED_UPDATE,\n        '*': 0,\n        '=': HEAD_UPTODATE,\n        ' ': FAST_FORWARD,\n        '-': TAG_UPDATE,\n    }\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        HEAD_UPTODATE: 'uptodate',\n        TAG_UPDATE: 'tag-update',\n        REJECTED: 'rejected',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Parse information from the given line as returned by git-fetch -v\n        and return a new FetchInfo object representing this information.\n        \"\"\"\n        match = cls._re_fetch_result.match(line)\n        if match is None:\n            raise ValueError(\"Failed to parse line: %r\" % line)\n\n        # parse lines\n        control_character, operation, local_remote_ref, remote_local_ref, note = \\\n            match.groups()\n\n        # parse flags from control_character\n        flags = 0\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\n                \"Control character %r unknown as parsed from line %r\"\n                % (control_character, line))\n        # END control char exception handling\n\n        # parse operation string for more info - makes no sense for symbolic refs,\n        # but we parse it anyway\n        old_commit = None\n        if 'rejected' in operation:\n            flags |= cls.REJECTED\n        if 'new tag' in operation:\n            flags |= cls.NEW_TAG\n        if 'tag update' in operation:\n            flags |= cls.TAG_UPDATE\n        if 'new branch' in operation:\n            flags |= cls.NEW_HEAD\n        if '...' in operation or '..' in operation:\n            split_token = '...'\n            if control_character == ' ':\n                split_token = split_token[:-1]\n            old_commit = operation.split(split_token)[0]\n        # END handle refspec\n\n        return cls(\n            ref=remote_local_ref.strip(),\n            local_ref=local_remote_ref.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=note,\n            old_commit=old_commit,\n        )\n\n\nclass PushInfo(dict):\n    \"\"\"dict that carries results of a push operation of a single head\n\n    Reduced variant of GitPython's RemoteProgress class\n\n    Original copyright:\n        Copyright (C) 2008, 2009 Michael Trier and contributors\n    Original license:\n        BSD 3-Clause \"New\" or \"Revised\" License\n    \"\"\"\n    NEW_TAG, NEW_HEAD, NO_MATCH, REJECTED, REMOTE_REJECTED, REMOTE_FAILURE, DELETED, \\\n        FORCED_UPDATE, FAST_FORWARD, UP_TO_DATE, ERROR = [1 << x for x in range(11)]\n\n    _flag_map = {'X': NO_MATCH,\n                 '-': DELETED,\n                 '*': 0,\n                 '+': FORCED_UPDATE,\n                 ' ': FAST_FORWARD,\n                 '=': UP_TO_DATE,\n                 '!': ERROR}\n\n    _operation_map = {\n        NEW_TAG: 'new-tag',\n        NEW_HEAD: 'new-branch',\n        NO_MATCH: 'no-match',\n        REJECTED: 'rejected',\n        REMOTE_REJECTED: 'remote-rejected',\n        REMOTE_FAILURE: 'remote-failure',\n        DELETED: 'deleted',\n        FORCED_UPDATE: 'forced-update',\n        FAST_FORWARD: 'fast-forward',\n        UP_TO_DATE: 'uptodate',\n        ERROR: 'error',\n    }\n\n    @classmethod\n    def _from_line(cls, line):\n        \"\"\"Create a new PushInfo instance as parsed from line which is expected to be like\n            refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes\"\"\"\n        control_character, from_to, summary = line.split('\\t', 3)\n        flags = 0\n\n        # control character handling\n        try:\n            flags |= cls._flag_map[control_character]\n        except KeyError:\n            raise ValueError(\"Control character %r unknown as parsed from line %r\" % (control_character, line))\n        # END handle control character\n\n        # from_to handling\n        from_ref_string, to_ref_string = from_to.split(':')\n\n        # commit handling, could be message or commit info\n        old_commit = None\n        if summary.startswith('['):\n            if \"[rejected]\" in summary:\n                flags |= cls.REJECTED\n            elif \"[remote rejected]\" in summary:\n                flags |= cls.REMOTE_REJECTED\n            elif \"[remote failure]\" in summary:\n                flags |= cls.REMOTE_FAILURE\n            elif \"[no match]\" in summary:\n                flags |= cls.ERROR\n            elif \"[new tag]\" in summary:\n                flags |= cls.NEW_TAG\n            elif \"[new branch]\" in summary:\n                flags |= cls.NEW_HEAD\n            # uptodate encoded in control character\n        else:\n            # fast-forward or forced update - was encoded in control character,\n            # but we parse the old and new commit\n            split_token = \"...\"\n            if control_character == \" \":\n                split_token = \"..\"\n            old_sha, _new_sha = summary.split(' ')[0].split(split_token)\n            # have to use constructor here as the sha usually is abbreviated\n            old_commit = old_sha\n        # END message handling\n\n        return cls(\n            from_ref=from_ref_string.strip(),\n            to_ref=to_ref_string.strip(),\n            # convert flag int into a list of operation labels\n            operations=[\n                cls._operation_map[o]\n                for o in cls._operation_map.keys()\n                if flags & o\n            ],\n            note=summary.strip(),\n            old_commit=old_commit,\n        )\n\n\n# Compatibility kludge.  See GitRepo.get_submodules().\nSubmodule = namedtuple(\"Submodule\", [\"name\", \"path\", \"url\"])\n\n\nclass GitRepo(RepoInterface, metaclass=PathBasedFlyweight):\n    \"\"\"Representation of a git repository\n\n    \"\"\"\n    # We must check git config to have name and email set, but\n    # should do it once\n    _config_checked = False\n\n    # Begin Flyweight:\n\n    _unique_instances = WeakValueDictionary()\n\n    def _flyweight_invalid(self):\n        return not self.is_valid_git()\n\n    @classmethod\n    def _flyweight_reject(cls, id_, *args, **kwargs):\n        # TODO:\n        # This is a temporary approach. See PR # ...\n        # create = kwargs.pop('create', None)\n        # kwargs.pop('path', None)\n        # if create and kwargs:\n        #     # we have `create` plus options other than `path`\n        #     return \"Call to {0}() with args {1} and kwargs {2} conflicts \" \\\n        #            \"with existing instance {3}.\" \\\n        #            \"This is likely to be caused by inconsistent logic in \" \\\n        #            \"your code.\" \\\n        #            \"\".format(cls, args, kwargs, cls._unique_instances[id_])\n        pass\n\n    # End Flyweight\n\n    def __hash__(self):\n        # the flyweight key is already determining unique instances\n        # add the class name to distinguish from strings of a path\n        return hash((self.__class__.__name__, self.__weakref__.key))\n\n    # This is the least common denominator to claim that a user\n    # used DataLad.\n    # For now citing Zenodo's all (i.e., latest) version\n    @due.dcite(Doi(\"10.5281/zenodo.808846\"),\n               # override path since there is no need ATM for such details\n               path=\"datalad\",\n               description=\"DataLad - Data management and distribution platform\")\n    def __init__(self, path, url=None, runner=None, create=True,\n                 git_opts=None, repo=None, fake_dates=False,\n                 create_sanity_checks=True,\n                 **kwargs):\n        \"\"\"Creates representation of git repository at `path`.\n\n        Can also be used to create a git repository at `path`.\n\n        Parameters\n        ----------\n        path: str\n          path to the git repository; In case it's not an absolute path,\n          it's relative to PWD\n        url: str, optional\n          DEPRECATED -- use .clone() class method\n          url to the to-be-cloned repository. Requires a valid git url\n          according to:\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        create: bool, optional\n          if true, creates a git repository at `path` if there is none. Also\n          creates `path`, if it doesn't exist.\n          If set to false, an exception is raised in case `path` doesn't exist\n          or doesn't contain a git repository.\n        repo: git.Repo, optional\n          GitPython's Repo instance to (re)use if provided\n        create_sanity_checks: bool, optional\n          Whether to perform sanity checks during initialization (when\n          `create=True` and target path is not a valid repo already), such as\n          that new repository is not created in the directory where git already\n          tracks some files.\n        kwargs:\n          keyword arguments serving as additional options to the git-init\n          command. Therefore, it makes sense only if called with `create`.\n\n          Generally, this way of passing options to the git executable is\n          (or will be) used a lot in this class. It's a transformation of\n          python-style keyword arguments (or a `dict`) to command line arguments,\n          provided by GitPython.\n\n          A single character keyword will be prefixed by '-', multiple characters\n          by '--'. An underscore in the keyword becomes a dash. The value of the\n          keyword argument is used as the value for the corresponding command\n          line argument. Assigning a boolean creates a flag.\n\n          Examples:\n          no_commit=True => --no-commit\n          C='/my/path'   => -C /my/path\n\n        \"\"\"\n\n        # BEGIN Repo validity test\n        # We want to fail early for tests, that would be performed a lot. In particular this is about\n        # GitRepo.is_valid_repo. We would use the latter to decide whether or not to call GitRepo() only for\n        # __init__ to then test the same things again. If we fail early we can save the additional test from outer\n        # scope.\n        self.path = path\n\n        # Note, that the following three path objects are used often and therefore\n        # are stored for performance. Path object creation comes with a cost. Most noteably,\n        # this is used for validity checking of the repository.\n        self.pathobj = ut.Path(self.path)\n        self.dot_git = self.pathobj / '.git'\n        # Read a potential .git file in order to not do that over and over again, when testing is_valid_git() etc.\n        # TODO: There's still some code duplication with static method GitRepo.get_git_dir()\n        #       However, it's returning relative path. So, the logic in usage needs to be unified in order to melt both\n        #       pieces.\n        if self.dot_git.is_file():\n            with self.dot_git.open() as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    self.dot_git = self.pathobj / line[7:].strip()\n                else:\n                    raise InvalidGitRepositoryError(\"Invalid .git file\")\n\n        elif self.dot_git.is_symlink():\n            self.dot_git = self.dot_git.resolve()\n\n        self._valid_git_test_path = self.dot_git / 'HEAD'\n        _valid_repo = self.is_valid_git()\n\n        do_create = False\n        if create and not _valid_repo:\n            if repo is not None:\n                # `repo` passed with `create`, which doesn't make sense\n                raise TypeError(\"argument 'repo' must not be used with 'create'\")\n            do_create = True\n        else:\n            # Note: We used to call gitpy.Repo(path) here, which potentially\n            # raised NoSuchPathError or InvalidGitRepositoryError. This is\n            # used by callers of GitRepo.__init__() to detect whether we have a\n            # valid repo at `path`. Now, with switching to lazy loading property\n            # `repo`, we detect those cases without instantiating a\n            # gitpy.Repo().\n\n            if not exists(path):\n                raise NoSuchPathError(path)\n            if not _valid_repo:\n                raise InvalidGitRepositoryError(path)\n        # END Repo validity test\n\n        if url is not None:\n            raise DeprecatedError(\n                new=\".clone() class method\",\n                version=\"0.5.0\",\n                msg=\"RF: url passed to init()\"\n            )\n\n        # So that we \"share\" control paths with git/git-annex\n        if ssh_manager:\n            ssh_manager.assure_initialized()\n\n        # note: we may also want to distinguish between a path to the worktree\n        # and the actual repository\n\n        # Could be used to e.g. disable automatic garbage and autopacking\n        # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0']\n        self._GIT_COMMON_OPTIONS = []\n        # actually no need with default GitPython db backend not in memory\n        # default_git_odbt but still allows for faster testing etc.\n        # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = []\n\n        if git_opts is None:\n            git_opts = {}\n        if kwargs:\n            git_opts.update(kwargs)\n\n        self.cmd_call_wrapper = runner or GitRunner(cwd=self.path)\n        self._repo = repo\n        self._cfg = None\n\n        if do_create:  # we figured it out ealier\n            self._repo = self._create_empty_repo(path,\n                                                 create_sanity_checks, **git_opts)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # calls above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        # with DryRunProtocol path might still not exist\n        if exists(self.path):\n            self.inode = os.stat(self.path).st_ino\n        else:\n            self.inode = None\n\n        if fake_dates:\n            self.configure_fake_dates()\n        # Set by fake_dates_enabled to cache config value across this instance.\n        self._fake_dates_enabled = None\n\n    def _create_empty_repo(self, path, sanity_checks=True, **kwargs):\n        if not op.lexists(path):\n            os.makedirs(path)\n        elif sanity_checks and external_versions['cmd:git'] < '2.14.0':\n            warnings.warn(\n                \"Your git version (%s) is too old, we will not safe-guard \"\n                \"against creating a new repository under already known to git \"\n                \"subdirectory\" % external_versions['cmd:git'],\n                OutdatedExternalDependencyWarning\n            )\n        elif sanity_checks:\n            # Verify that we are not trying to initialize a new git repository\n            # under a directory some files of which are already tracked by git\n            # use case: https://github.com/datalad/datalad/issues/3068\n            try:\n                stdout, _ = self._git_custom_command(\n                    None, ['git', 'ls-files'], cwd=path, expect_fail=True\n                )\n                if stdout:\n                    raise PathKnownToRepositoryError(\n                        \"Failing to initialize new repository under %s where \"\n                        \"following files are known to a repository above: %s\"\n                        % (path, stdout)\n                    )\n            except CommandError:\n                # assume that all is good -- we are not under any repo\n                pass\n\n        cmd = ['git', 'init']\n        cmd.extend(kwargs.pop('_from_cmdline_', []))\n        cmd.extend(to_options(**kwargs))\n        lgr.debug(\n            \"Initialize empty Git repository at '%s'%s\",\n            path,\n            ' %s' % cmd[2:] if cmd[2:] else '')\n\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                cmd,\n                cwd=path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            lgr.error(exc_str(exc))\n            raise\n        # we want to return None and have lazy eval take care of\n        # the rest\n        return\n\n    @property\n    def repo(self):\n        # with DryRunProtocol path not exist\n        if exists(self.path):\n            inode = os.stat(self.path).st_ino\n        else:\n            inode = None\n        if self.inode != inode:\n\n            # TODO: - what if we never had any inode? possible?\n            #       - also: call to self._repo and only afterwards checking whether it's None seems strange\n\n            # reset background processes invoked by GitPython:\n            # it can be that we never got to instantiate a GitPython repo instance\n            if self._repo:\n                self._repo.git.clear_cache()\n            self.inode = inode\n\n        if self._repo is None:\n            # Note, that this may raise GitCommandError, NoSuchPathError,\n            # InvalidGitRepositoryError:\n            self._repo = self.cmd_call_wrapper(\n                Repo,\n                self.path)\n            lgr.log(8, \"Using existing Git repository at %s\", self.path)\n\n        # inject git options into GitPython's git call wrapper:\n        # Note: `None` currently can happen, when Runner's protocol prevents\n        # call of Repo(path) above from being actually executed (DryRunProtocol)\n        if self._repo is not None:\n            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS\n\n        return self._repo\n\n    @classmethod\n    def clone(cls, url, path, *args, clone_options=None, **kwargs):\n        \"\"\"Clone url into path\n\n        Provides workarounds for known issues (e.g.\n        https://github.com/datalad/datalad/issues/785)\n\n        Parameters\n        ----------\n        url : str\n        path : str\n        clone_options : dict\n          Key/value pairs of arbitrary options that will be passed on to the\n          underlying call to `git-clone`.\n        expect_fail : bool\n          Whether expect that command might fail, so error should be logged then\n          at DEBUG level instead of ERROR\n        \"\"\"\n\n        if 'repo' in kwargs:\n            raise TypeError(\"argument 'repo' conflicts with cloning\")\n            # TODO: what about 'create'?\n\n        expect_fail = kwargs.pop('expect_fail', False)\n        # fail early on non-empty target:\n        from os import listdir\n        if exists(path) and listdir(path):\n            # simulate actual GitCommandError:\n            lgr.warning(\"destination path '%s' already exists and is not an \"\n                        \"empty directory.\" % path)\n            raise GitCommandError(\n                ['git', 'clone', '-v', url, path],\n                128,\n                \"fatal: destination path '%s' already exists and is not an \"\n                \"empty directory.\" % path)\n        else:\n            # protect against cloning into existing and obviously dangling\n            # instance for that location\n            try:\n                del cls._unique_instances[path]\n            except KeyError:\n                # didn't exist - all fine\n                pass\n\n        # Massage URL\n        url_ri = RI(url) if not isinstance(url, RI) else url\n        if not on_windows:\n            # if we are on windows, the local path of a URL\n            # would not end up being a proper local path and cloning\n            # would fail. Don't try to be smart and just pass the\n            # URL along unmodified\n\n            # try to get a local path from `url`:\n            try:\n                url = url_ri.localpath\n                url_ri = RI(url)\n            except ValueError:\n                pass\n\n        if is_ssh(url_ri):\n            ssh_manager.get_connection(url).open()\n        else:\n            if isinstance(url_ri, PathRI):\n                # expand user, because execution not going through a shell\n                # doesn't work well otherwise\n                new_url = os.path.expanduser(url)\n                if url != new_url:\n                    lgr.info(\"Expanded source path to %s from %s\", new_url, url)\n                    url = new_url\n\n        fix_annex = None\n        ntries = 5  # 3 is not enough for robust workaround\n        for trial in range(ntries):\n            try:\n                lgr.debug(\"Git clone from {0} to {1}\".format(url, path))\n\n                out, err = WitlessRunner(\n                    env=GitRunner.get_git_environ_adjusted()).run(\n                        ['git', 'clone', '--progress', url, path] \\\n                        + (to_options(**clone_options)\n                           if clone_options else []),\n                        protocol=GitProgress,\n                )\n                # fish out non-critical warnings by git-clone\n                # (empty repo clone, etc.), all other content is logged\n                # by the progress helper to 'debug'\n                for errline in err.splitlines():\n                    if errline.startswith('warning:'):\n                        lgr.warning(errline[8:].strip())\n                lgr.debug(\"Git clone completed\")\n                break\n            except CommandError as e:\n                # log here but let caller decide what to do\n                e_str = exc_str(e)\n                # see https://github.com/datalad/datalad/issues/785\n                if re.search(\"Request for .*aborted.*Unable to find\", str(e),\n                             re.DOTALL) \\\n                        and trial < ntries - 1:\n                    lgr.info(\n                        \"Hit a known issue with Git (see GH#785). Trial #%d, \"\n                        \"retrying\",\n                        trial)\n                    continue\n                    (lgr.debug if expect_fail else lgr.error)(e_str)\n\n                if \"Clone succeeded, but checkout failed.\" in str(e):\n                    fix_annex = e\n                    break\n\n                raise\n\n        # get ourselves a repository instance\n        gr = cls(path, *args, **kwargs)\n        if fix_annex:\n            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then\n            # would need to import our own subclass\n            if hasattr(gr, 'is_valid_annex'):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                if not gr.is_initialized():\n                    gr._init()\n                gr.fsck()\n            else:\n                lgr.warning(\"Experienced issues while cloning: %s\", exc_str(fix_annex))\n        return gr\n\n    def __del__(self):\n        # unbind possibly bound ConfigManager, to prevent all kinds of weird\n        # stalls etc\n        self._cfg = None\n        # Make sure to flush pending changes, especially close batch processes\n        # (internal `git cat-file --batch` by GitPython)\n        try:\n            if getattr(self, '_repo', None) is not None and exists(self.path):\n                # gc might be late, so the (temporary)\n                # repo doesn't exist on FS anymore\n                self._repo.git.clear_cache()\n                # We used to write out the index to flush GitPython's\n                # state... but such unconditional write is really a workaround\n                # and does not play nice with read-only operations - permission\n                # denied etc. So disabled \n                #if exists(opj(self.path, '.git')):  # don't try to write otherwise\n                #    self.repo.index.write()\n        except (InvalidGitRepositoryError, AttributeError):\n            # might have being removed and no longer valid or attributes unbound\n            pass\n\n    def __repr__(self):\n        return \"<GitRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def __eq__(self, obj):\n        \"\"\"Decides whether or not two instances of this class are equal.\n\n        This is done by comparing the base repository path.\n        \"\"\"\n        return self.path == obj.path\n\n    def is_valid_git(self):\n        \"\"\"Returns whether the underlying repository appears to be still valid\n\n        Note, that this almost identical to the classmethod is_valid_repo(). However,\n        if we are testing an existing instance, we can save Path object creations. Since this testing\n        is done a lot, this is relevant. Creation of the Path objects in is_valid_repo() takes nearly half the time of\n        the entire function.\n\n        Also note, that this method is bound to an instance but still class-dependent, meaning that a subclass\n        cannot simply overwrite it. This is particularly important for the call from within __init__(),\n        which in turn is called by the subclasses' __init__. Using an overwrite would lead to the wrong thing being\n        called.\n        \"\"\"\n\n        return self.dot_git.exists() and (\n                not self.dot_git.is_dir() or self._valid_git_test_path.exists()\n        )\n\n    @classmethod\n    def is_valid_repo(cls, path):\n        \"\"\"Returns if a given path points to a git repository\"\"\"\n        if not isinstance(path, Path):\n            path = Path(path)\n        dot_git_path = path / '.git'\n\n        # the aim here is to have this test as cheap as possible, because\n        # it is performed a lot\n        # recognize two things as good-enough indicators of a present\n        # repo: 1) a non-empty .git directory (#3473)\n        #          NOTE: It's actually faster (and more accurate) to test for existence of a particular subpath.\n        #                This should be something that's there right after git-init. Going for .git/HEAD ATM.\n        #\n        #                In [11]: %timeit path.exists() and (not path.is_dir() or head_path.exists())\n        #                4.93 \u00b5s \u00b1 34.8 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n        #                In [12]: %timeit path.exists() and (not path.is_dir() or any(path.iterdir()))\n        #                12.8 \u00b5s \u00b1 150 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n        #\n        #   and 2) a pointer file or symlink\n\n        return dot_git_path.exists() and (\n            not dot_git_path.is_dir() or (dot_git_path / 'HEAD').exists()\n        )\n\n    @staticmethod\n    def get_git_dir(repo):\n        \"\"\"figure out a repo's gitdir\n\n        '.git' might be a  directory, a symlink or a file\n\n        Note\n        ----\n        Please try using GitRepo.dot_git instead! That one's not static, but it's cheaper and you should avoid\n        not having an instance of a repo you're working on anyway. Note, that the property in opposition to this method\n        returns an absolute path.\n\n\n        Parameters\n        ----------\n        repo: path or Repo instance\n          currently expected to be the repos base dir\n\n        Returns\n        -------\n        str\n          relative path to the repo's git dir; So, default would be \".git\"\n        \"\"\"\n        if hasattr(repo, 'path'):\n            # repo instance like given\n            repo = repo.path\n        dot_git = op.join(repo, \".git\")\n        if not op.exists(dot_git):\n            raise RuntimeError(\"Missing .git in %s.\" % repo)\n        elif op.islink(dot_git):\n            git_dir = os.readlink(dot_git)\n        elif op.isdir(dot_git):\n            git_dir = \".git\"\n        elif op.isfile(dot_git):\n            with open(dot_git) as f:\n                git_dir = f.readline()\n                if git_dir.startswith(\"gitdir:\"):\n                    git_dir = git_dir[7:]\n                git_dir = git_dir.strip()\n\n        return git_dir\n\n    @property\n    def config(self):\n        \"\"\"Get an instance of the parser for the persistent repository\n        configuration.\n\n        Note: This allows to also read/write .datalad/config,\n        not just .git/config\n\n        Returns\n        -------\n        ConfigManager\n        \"\"\"\n        if self._cfg is None:\n            # associate with this dataset and read the entire config hierarchy\n            self._cfg = ConfigManager(dataset=self, source='any')\n        return self._cfg\n\n    def is_with_annex(self):\n        \"\"\"Report if GitRepo (assumed) has (remotes with) a git-annex branch\n        \"\"\"\n        return any(\n            b['refname:strip=2'] == 'git-annex' or b['refname:strip=2'].endswith('/git-annex')\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern=['refs/heads', 'refs/remotes'])\n        )\n\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n        cmd = ['git']\n        if git_options:\n            cmd.extend(git_options)\n        cmd += [\"rev-parse\", \"--show-toplevel\"]\n        try:\n            toppath, err = GitRunner().run(\n                cmd,\n                cwd=path,\n                log_stdout=True, log_stderr=True,\n                expect_fail=True, expect_stderr=True)\n            toppath = toppath.rstrip('\\n\\r')\n        except CommandError:\n            return None\n        except OSError:\n            toppath = GitRepo.get_toppath(dirname(path), follow_up=follow_up,\n                                          git_options=git_options)\n\n        if follow_up:\n            path_ = path\n            path_prev = \"\"\n            while path_ and path_ != path_prev:  # on top /.. = /\n                if realpath(path_) == toppath:\n                    toppath = path_\n                    break\n                path_prev = path_\n                path_ = dirname(path_)\n\n        return toppath\n\n    @normalize_paths\n    def add(self, files, git=True, git_options=None, update=False):\n        \"\"\"Adds file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list\n          list of paths to add\n        git: bool\n          somewhat ugly construction to be compatible with AnnexRepo.add();\n          has to be always true.\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n        Returns\n        -------\n        list\n          Of status dicts.\n        \"\"\"\n        # under all circumstances call this class' add_ (otherwise\n        # AnnexRepo.add would go into a loop\n        return list(GitRepo.add_(self, files, git=git, git_options=git_options,\n                    update=update))\n\n    def add_(self, files, git=True, git_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        # TODO: git_options is used as options for the git-add here,\n        # instead of options to the git executable => rename for consistency\n\n        if not git:\n            lgr.warning(\n                'GitRepo.add() called with git=%s, this should not happen',\n                git)\n            git = True\n\n        # there is no other way then to collect all files into a list\n        # at this point, because we need to pass them at once to a single\n        # `git add` call\n        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]\n\n        if not (files or git_options or update):\n            # wondering why just a warning? in cmdline this is also not an error\n            lgr.warning(\"add was called with empty file list and no options.\")\n            return\n\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                files,\n                # Set annex.largefiles to prevent storing files in annex when\n                # GitRepo() is instantiated with a v6+ annex repo.\n                ['git', '-c', 'annex.largefiles=nothing', 'add'] +\n                ensure_list(git_options) +\n                to_options(update=update) + ['--verbose']\n            )\n            # get all the entries\n            for o in self._process_git_get_output(*add_out):\n                yield o\n            # Note: as opposed to git cmdline, force is True by default in\n            #       gitpython, which would lead to add things, that are\n            #       ignored or excluded otherwise\n            # 2. Note: There is an issue with globbing (like adding '.'),\n            #       which apparently doesn't care for 'force' and therefore\n            #       adds '.git/...'. May be it's expanded at the wrong\n            #       point in time or sth. like that.\n            # For now, use direct call to git add.\n            #self.cmd_call_wrapper(self.repo.index.add, files, write=True,\n            #                      force=False)\n            # TODO: May be make use of 'fprogress'-option to indicate\n            # progress\n            # But then, we don't have it for git-annex add, anyway.\n            #\n            # TODO: Is write=True a reasonable way to do it?\n            # May be should not write until success of operation is\n            # confirmed?\n            # What's best in case of a list of files?\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n        # Make sure return value from GitRepo is consistent with AnnexRepo\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return\n\n    @staticmethod\n    def _process_git_get_output(stdout, stderr=None):\n        \"\"\"Given both outputs (stderr is ignored atm) of git add - process it\n\n        Primarily to centralize handling in both indirect annex and direct\n        modes when ran through proxy\n        \"\"\"\n        return [{u'file': f, u'success': True}\n                for f in re.findall(\"'(.*)'[\\n$]\", ensure_unicode(stdout))]\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, recursive=False, **kwargs):\n        \"\"\"Remove files.\n\n        Calls git-rm.\n\n        Parameters\n        ----------\n        files: str\n          list of paths to remove\n        recursive: False\n          whether to allow recursive removal from subdirectories\n        kwargs:\n          see `__init__`\n\n        Returns\n        -------\n        [str]\n          list of successfully removed files.\n        \"\"\"\n\n        files = _remove_empty_items(files)\n\n        if recursive:\n            kwargs['r'] = True\n        stdout, stderr = self._git_custom_command(\n            files, ['git', 'rm'] + to_options(**kwargs))\n\n        # output per removed file is expected to be \"rm 'PATH'\":\n        return [line.strip()[4:-1] for line in stdout.splitlines()]\n\n        #return self.repo.git.rm(files, cached=False, **kwargs)\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks\n        \"\"\"\n        # All GitPython commands should take care about flushing index\n        # whenever they modify it, so we would not care to do anything\n        # if self.repo is not None and exists(opj(self.path, '.git')):  # don't try to write otherwise:\n        #     # flush possibly cached in GitPython changes to index:\n        #     # if self.repo.git:\n        #     #     sys.stderr.write(\"CLEARING\\n\")\n        #     #     self.repo.git.clear_cache()\n        #     self.repo.index.write()\n\n        # Close batched by GitPython git processes etc\n        # Ref: https://github.com/gitpython-developers/GitPython/issues/718\n        self.repo.__del__()\n        pass\n\n    @staticmethod\n    def _get_prefixed_commit_msg(msg):\n        DATALAD_PREFIX = \"[DATALAD]\"\n        return DATALAD_PREFIX if not msg else \"%s %s\" % (DATALAD_PREFIX, msg)\n\n    def for_each_ref_(self, fields=('objectname', 'objecttype', 'refname'),\n                      pattern=None, points_at=None, sort=None, count=None,\n                      contains=None):\n        \"\"\"Wrapper for `git for-each-ref`\n\n        Please see manual page git-for-each-ref(1) for a complete overview\n        of its functionality. Only a subset of it is supported by this\n        wrapper.\n\n        Parameters\n        ----------\n        fields : iterable or str\n          Used to compose a NULL-delimited specification for for-each-ref's\n          --format option. The default field list reflects the standard\n          behavior of for-each-ref when the --format option is not given.\n        pattern : list or str, optional\n          If provided, report only refs that match at least one of the given\n          patterns.\n        points_at : str, optional\n          Only list refs which points at the given object.\n        sort : list or str, optional\n          Field name(s) to sort-by. If multiple fields are given, the last one\n          becomes the primary key. Prefix any field name with '-' to sort in\n          descending order.\n        count : int, optional\n          Stop iteration after the given number of matches.\n        contains : str, optional\n          Only list refs which contain the specified commit.\n\n        Yields\n        ------\n        dict with items matching the given `fields`\n\n        Raises\n        ------\n        ValueError\n          if no `fields` are given\n\n        RuntimeError\n          if `git for-each-ref` returns a record where the number of\n          properties does not match the number of `fields`\n        \"\"\"\n        if not fields:\n            raise ValueError('no `fields` provided, refuse to proceed')\n        fields = ensure_list(fields)\n        cmd = [\n            \"git\",\n            \"for-each-ref\",\n            \"--format={}\".format(\n                '%00'.join(\n                    '%({})'.format(f) for f in fields)),\n        ]\n        if points_at:\n            cmd.append('--points-at={}'.format(points_at))\n        if contains:\n            cmd.append('--contains={}'.format(contains))\n        if sort:\n            for k in ensure_list(sort):\n                cmd.append('--sort={}'.format(k))\n        if pattern:\n            cmd += ensure_list(pattern)\n        if count:\n            cmd.append('--count={:d}'.format(count))\n\n        out, _ = self._git_custom_command(None, cmd)\n        for line in out.splitlines():\n            props = line.split('\\0')\n            if len(fields) != len(props):\n                raise RuntimeError(\n                    'expected fields {} from git-for-each-ref, but got: {}'.format(\n                        fields, props))\n            yield dict(zip(fields, props))\n\n    def configure_fake_dates(self):\n        \"\"\"Configure repository to use fake dates.\n        \"\"\"\n        lgr.debug(\"Enabling fake dates\")\n        self.config.set(\"datalad.fake-dates\", \"true\")\n\n    @property\n    def fake_dates_enabled(self):\n        \"\"\"Is the repository configured to use fake dates?\n        \"\"\"\n        if self._fake_dates_enabled is None:\n            self._fake_dates_enabled = \\\n                self.config.getbool('datalad', 'fake-dates', default=False)\n        return self._fake_dates_enabled\n\n    def add_fake_dates(self, env):\n        \"\"\"Add fake dates to `env`.\n\n        Parameters\n        ----------\n        env : dict or None\n            Environment variables.\n\n        Returns\n        -------\n        A dict (copied from env), with date-related environment\n        variables for git and git-annex set.\n        \"\"\"\n        env = (env if env is not None else os.environ).copy()\n        # Note: Use _git_custom_command here rather than repo.git.for_each_ref\n        # so that we use annex-proxy in direct mode.\n        last_date = list(self.for_each_ref_(\n            fields='committerdate:raw',\n            count=1,\n            pattern='refs/heads',\n            sort=\"-committerdate\",\n        ))\n\n        if last_date:\n            # Drop the \"contextual\" timezone, leaving the unix timestamp.  We\n            # avoid :unix above because it wasn't introduced until Git v2.9.4.\n            last_date = last_date[0]['committerdate:raw'].split()[0]\n            seconds = int(last_date)\n        else:\n            seconds = self.config.obtain(\"datalad.fake-dates-start\")\n        seconds_new = seconds + 1\n        date = \"@{} +0000\".format(seconds_new)\n\n        lgr.debug(\"Setting date to %s\",\n                  time.strftime(\"%a %d %b %Y %H:%M:%S +0000\",\n                                time.gmtime(seconds_new)))\n\n        env[\"GIT_AUTHOR_DATE\"] = date\n        env[\"GIT_COMMITTER_DATE\"] = date\n        env[\"GIT_ANNEX_VECTOR_CLOCK\"] = str(seconds_new)\n\n        return env\n\n    def commit(self, msg=None, options=None, _datalad_msg=False, careless=True,\n               files=None, date=None, index_file=None):\n        \"\"\"Commit changes to git.\n\n        Parameters\n        ----------\n        msg: str, optional\n          commit-message\n        options: list of str, optional\n          cmdline options for git-commit\n        _datalad_msg: bool, optional\n          To signal that commit is automated commit by datalad, so\n          it would carry the [DATALAD] prefix\n        careless: bool, optional\n          if False, raise when there's nothing actually committed;\n          if True, don't care\n        files: list of str, optional\n          path(s) to commit\n        date: str, optional\n          Date in one of the formats git understands\n        index_file: str, optional\n          An alternative index to use\n        \"\"\"\n\n        self.precommit()\n\n        # assemble commandline\n        cmd = ['git', 'commit']\n        options = ensure_list(options)\n\n        if date:\n            options += [\"--date\", date]\n\n        if not msg:\n            msg = 'Recorded changes'\n            _datalad_msg = True\n\n        if _datalad_msg:\n            msg = self._get_prefixed_commit_msg(msg)\n\n        options += [\"-m\", msg]\n        cmd.extend(options)\n\n        # set up env for commit\n        env = GitRunner.get_git_environ_adjusted()\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n        if index_file:\n            env['GIT_INDEX_FILE'] = index_file\n\n        lgr.debug(\"Committing via direct call of git: %s\" % cmd)\n\n        file_chunks = generate_file_chunks(files, cmd) if files else [[]]\n\n        runner = WitlessRunner(cwd=self.path, env=env)\n\n        try:\n            for i, chunk in enumerate(file_chunks):\n                cur_cmd = cmd + (\n                    # if this is an explicit dry-run, there is no point in\n                    # amending, because no commit was ever made\n                    # otherwise, amend the first commit, and prevent\n                    # leaving multiple commits behind\n                    ['--amend', '--no-edit']\n                    if i > 0 and '--dry-run' not in cmd\n                    else []\n                ) + ['--'] + chunk\n                runner.run(\n                    cur_cmd,\n                    protocol=StdOutErrCapture,\n                    stdin=None,\n                )\n        except CommandError as e:\n            # real errors first\n            if \"did not match any file(s) known to git\" in e.stderr:\n                raise FileNotInRepositoryError(\n                    cmd=e.cmd,\n                    msg=\"File(s) unknown to git\",\n                    code=e.code,\n                    filename=linesep.join([\n                        l for l in e.stderr.splitlines()\n                        if l.startswith(\"error: pathspec\")\n                    ])\n                )\n            # behavior choices now\n            elif not careless:\n                # not willing to compromise at all\n                raise\n            elif 'nothing to commit' in e.stdout:\n                lgr.debug(\"nothing to commit in %s. Ignored.\", self)\n            elif 'no changes added to commit' in e.stdout or \\\n                    'nothing added to commit' in e.stdout:\n                lgr.debug(\"no changes added to commit in %s. Ignored.\", self)\n            else:\n                raise\n\n    # TODO usage is primarily in the tests, consider making a test helper and\n    # remove from GitRepo API\n    def get_indexed_files(self):\n        \"\"\"Get a list of files in git's index\n\n        Returns\n        -------\n        list\n            list of paths rooting in git's base dir\n        \"\"\"\n\n        return [\n            str(r.relative_to(self.pathobj))\n            for r in self.get_content_info(\n                paths=None, ref=None, untracked='no', eval_file_type=False)\n        ]\n\n    def format_commit(self, fmt, commitish=None):\n        \"\"\"Return `git show` output for `commitish`.\n\n        Parameters\n        ----------\n        fmt : str\n            A format string accepted by `git show`.\n        commitish: str, optional\n          Any commit identifier (defaults to \"HEAD\").\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        cmd = ['git', 'show', '-z', '--no-patch', '--format=' + fmt]\n        if commitish is not None:\n            cmd.append(commitish + \"^{commit}\")\n        # make sure Git takes our argument as a revision\n        cmd.append('--')\n        try:\n            stdout, stderr = self._git_custom_command(\n                '', cmd, expect_stderr=True, expect_fail=True)\n        except CommandError as e:\n            if 'bad revision' in e.stderr:\n                raise ValueError(\"Unknown commit identifier: %s\" % commitish)\n            elif 'does not have any commits yet' in e.stderr:\n                return None\n            else:\n                raise e\n        # This trailing null is coming from the -z above, which avoids the\n        # newline that Git would append to the output. We could drop -z and\n        # strip the newline directly, but then we'd have to worry about\n        # compatibility across platforms.\n        return stdout.rsplit(\"\\0\", 1)[0]\n\n    def get_hexsha(self, commitish=None, short=False):\n        \"\"\"Return a hexsha for a given commitish.\n\n        Parameters\n        ----------\n        commitish : str, optional\n          Any identifier that refers to a commit (defaults to \"HEAD\").\n        short : bool, optional\n          Return the abbreviated form of the hexsha.\n\n        Returns\n        -------\n        str or, if there are not commits yet, None.\n        \"\"\"\n        stdout = self.format_commit(\"%{}\".format('h' if short else 'H'),\n                                    commitish)\n        if stdout is not None:\n            stdout = stdout.splitlines()\n            assert(len(stdout) == 1)\n            return stdout[0]\n\n    @normalize_paths(match_return_type=False)\n    def get_last_commit_hexsha(self, files):\n        \"\"\"Return the hash of the last commit the modified any of the given\n        paths\"\"\"\n        try:\n            commit = self.call_git(\n                ['rev-list', '-n1', 'HEAD'],\n                files=files,\n                expect_fail=True,\n            )\n            commit = commit.strip()\n            return commit if commit else None\n        except CommandError:\n            if self.get_hexsha() is None:\n                # unborn branch, don't freak out\n                return None\n            raise\n\n    def get_revisions(self, revrange=None, fmt=\"%H\", options=None):\n        \"\"\"Return list of revisions in `revrange`.\n\n        Parameters\n        ----------\n        revrange : str or list of str or None, optional\n            Revisions or revision ranges to walk. If None, revision defaults to\n            HEAD unless a revision-modifying option like `--all` or\n            `--branches` is included in `options`.\n        fmt : string, optional\n            Format accepted by `--format` option of `git log`. This should not\n            contain new lines because the output is split on new lines.\n        options : list of str, optional\n            Options to pass to `git log`.  This should not include `--format`.\n\n        Returns\n        -------\n        List of revisions (str), formatted according to `fmt`.\n        \"\"\"\n        if revrange is None:\n            revrange = []\n        elif isinstance(revrange, str):\n            revrange = [revrange]\n\n        cmd = [\"git\", \"log\", \"--format={}\".format(fmt)]\n        cmd.extend((options or []) + revrange + [\"--\"])\n        try:\n            stdout, _ = self._git_custom_command(None, cmd, expect_fail=True)\n        except CommandError as e:\n            if \"does not have any commits\" in e.stderr:\n                return []\n            raise\n        return stdout.splitlines()\n\n    def commit_exists(self, commitish):\n        \"\"\"Does `commitish` exist in the repo?\n\n        Parameters\n        ----------\n        commitish : str\n            A commit or an object that can be dereferenced to one.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        # Note: The peeling operator \"^{commit}\" is required so that rev-parse\n        # doesn't succeed if passed a full hexsha that is valid but doesn't\n        # exist.\n        return self.call_git_success(\n            [\"rev-parse\", \"--verify\", commitish + \"^{commit}\"])\n\n    def get_merge_base(self, commitishes):\n        \"\"\"Get a merge base hexsha\n\n        Parameters\n        ----------\n        commitishes: str or list of str\n          List of commitishes (branches, hexshas, etc) to determine the merge\n          base of. If a single value provided, returns merge_base with the\n          current branch.\n\n        Returns\n        -------\n        str or None\n          If no merge-base for given commits, or specified treeish doesn't\n          exist, None returned\n        \"\"\"\n        if isinstance(commitishes, str):\n            commitishes = [commitishes]\n        if not commitishes:\n            raise ValueError(\"Provide at least a single value\")\n        elif len(commitishes) == 1:\n            commitishes = commitishes + [self.get_active_branch()]\n\n        try:\n            base = self.call_git_oneline(['merge-base'] + commitishes)\n        except CommandError as exc:\n            if exc.code == 1 and not (exc.stdout or exc.stderr):\n                # No merge base was found (unrelated commits).\n                return None\n            if \"fatal: Not a valid object name\" in exc.stderr:\n                return None\n            raise\n\n        return base\n\n    def is_ancestor(self, reva, revb):\n        \"\"\"Is `reva` an ancestor of `revb`?\n\n        Parameters\n        ----------\n        reva, revb : str\n            Revisions.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        return self.call_git_success(\n            [\"merge-base\", \"--is-ancestor\", reva, revb])\n\n    def get_commit_date(self, branch=None, date='authored'):\n        \"\"\"Get the date stamp of the last commit (in a branch or head otherwise)\n\n        Parameters\n        ----------\n        date: {'authored', 'committed'}\n          Which date to return.  \"authored\" will be the date shown by \"git show\"\n          and the one possibly specified via --date to `git commit`\n\n        Returns\n        -------\n        int or None\n          None if no commit\n        \"\"\"\n        if date == 'committed':\n            format = '%ct'\n        elif date == 'authored':\n            format = '%at'\n        else:\n            raise ValueError('unknow date type: {}'.format(date))\n        d = self.format_commit(format, commitish=branch)\n        return int(d) if d else None\n\n    def get_active_branch(self):\n        \"\"\"Get the name of the active branch\n\n        Returns\n        -------\n        str or None\n          Returns None if there is no active branch, i.e. detached HEAD,\n          and the branch name otherwise.\n        \"\"\"\n        try:\n            out, _ = self._git_custom_command(\n                \"\", [\"git\", \"symbolic-ref\", \"HEAD\"],\n                expect_fail=True)\n        except CommandError as e:\n            if 'HEAD is not a symbolic ref' in e.stderr:\n                lgr.debug(\"detached HEAD in {0}\".format(self))\n                return None\n            else:\n                raise e\n        return out.strip()[11:]  # strip refs/heads/\n\n    def get_branches(self):\n        \"\"\"Get all branches of the repo.\n\n        Returns\n        -------\n        [str]\n            Names of all branches of this repository.\n        \"\"\"\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/heads')\n        ]\n\n    def get_remote_branches(self):\n        \"\"\"Get all branches of all remotes of the repo.\n\n        Returns\n        -----------\n        [str]\n            Names of all remote branches.\n        \"\"\"\n        # TODO: Reconsider melting with get_branches()\n\n        # TODO: treat entries like this: origin/HEAD -> origin/master'\n        # currently this is done in collection\n\n        return [\n            b['refname:strip=2']\n            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/remotes')\n        ]\n\n    def get_remotes(self, with_urls_only=False):\n        \"\"\"Get known remotes of the repository\n\n        Parameters\n        ----------\n        with_urls_only : bool, optional\n          return only remotes which have urls\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n\n        # Note: read directly from config and spare instantiation of gitpy.Repo\n        # since we need this in AnnexRepo constructor. Furthermore gitpy does it\n        # pretty much the same way and the use of a Repo instance seems to have\n        # no reason other than a nice object oriented look.\n        from datalad.utils import unique\n\n        self.config.reload()\n        remotes = unique([x[7:] for x in self.config.sections()\n                          if x.startswith(\"remote.\")])\n\n        if with_urls_only:\n            remotes = [\n                r for r in remotes\n                if self.config.get('remote.%s.url' % r)\n            ]\n        return remotes\n\n    # TODO this is practically unused outside the tests, consider turning\n    # into a test helper and trim from the API\n    def get_files(self, branch=None):\n        \"\"\"Get a list of files in git.\n\n        Lists the files in the (remote) branch.\n\n        Parameters\n        ----------\n        branch: str\n          Name of the branch to query. Default: active branch.\n\n        Returns\n        -------\n        [str]\n          list of files.\n        \"\"\"\n        return [\n            str(p.relative_to(self.pathobj))\n            for p in self.get_content_info(\n                paths=None, ref=branch, untracked='no', eval_file_type=False)\n            ]\n\n    @normalize_paths(match_return_type=False)\n    def _git_custom_command(self, files, cmd_str,\n                            log_stdout=True, log_stderr=True, log_online=False,\n                            expect_stderr=True, cwd=None, env=None,\n                            shell=None, expect_fail=False,\n                            check_fake_dates=False,\n                            index_file=None,\n                            updates_tree=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git commands\n        for proof of concept without the need to figure out, how this is done\n        via GitPython.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str or list\n          arbitrary command str. `files` is appended to that string.\n        updates_tree: bool\n          whether or not command updates the working tree. If True, triggers\n          necessary reevaluations like self.config.reload()\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n\n        # ensure cmd_str becomes a well-formed list:\n        if isinstance(cmd_str, str):\n            cmd = split_cmdline(cmd_str)\n        else:\n            cmd = cmd_str[:]  # we will modify in-place\n\n        assert(cmd[0] == 'git')\n        cmd = cmd[:1] + self._GIT_COMMON_OPTIONS + cmd[1:]\n\n        if check_fake_dates and self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if index_file:\n            env = (env if env is not None else os.environ).copy()\n            env['GIT_INDEX_FILE'] = index_file\n\n        # TODO?: wouldn't splitting interfer with above GIT_INDEX_FILE\n        #  handling????\n        try:\n            out, err = run_gitcommand_on_file_list_chunks(\n                self.cmd_call_wrapper.run,\n                cmd,\n                files,\n                log_stderr=log_stderr,\n                log_stdout=log_stdout,\n                log_online=log_online,\n                expect_stderr=expect_stderr,\n                cwd=cwd,\n                env=env,\n                shell=shell,\n                expect_fail=expect_fail)\n        except CommandError as e:\n            ignored = re.search(GitIgnoreError.pattern, e.stderr)\n            if ignored:\n                raise GitIgnoreError(cmd=e.cmd, msg=e.stderr,\n                                     code=e.code, stdout=e.stdout,\n                                     stderr=e.stderr,\n                                     paths=ignored.groups()[0].splitlines())\n            raise\n\n        if updates_tree:\n            lgr.debug(\"Reloading config due to supposed working tree update\")\n            self.config.reload()\n\n        return out, err\n\n    # Convenience wrappers for one-off git calls that don't require further\n    # processing or error handling.\n\n    def call_git(self, args, files=None,\n                 expect_stderr=False, expect_fail=False):\n        \"\"\"Call git and return standard output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        expect_fail : bool, optional\n          A non-zero exit is expected and should not be elevated above the\n          DEBUG level.\n\n        Returns\n        -------\n        standard output (str)\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          expect_fail=expect_fail,\n                                          check_fake_dates=True)\n        return out\n\n    def call_git_items_(self, args, files=None, expect_stderr=False, sep=None):\n        \"\"\"Call git, splitting output on `sep`.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Returns\n        -------\n        Generator that yields output items.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        \"\"\"\n        out, _ = self._git_custom_command(files, [\"git\"] + args,\n                                          expect_stderr=expect_stderr,\n                                          check_fake_dates=True)\n        yield from (out.split(sep) if sep else out.splitlines())\n\n    def call_git_oneline(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git for a single line of output.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n        sep : str, optional\n          Split the output by `str.split(sep)` rather than `str.splitlines`.\n\n        Raises\n        ------\n        CommandError if the call exits with a non-zero status.\n        AssertionError if there is more than one line of output.\n        \"\"\"\n        lines = list(self.call_git_items_(args, files=files,\n                                          expect_stderr=expect_stderr))\n        if len(lines) > 1:\n            raise AssertionError(\n                \"Expected {} to return single line, but it returned {}\"\n                .format([\"git\"] + args, lines))\n        return lines[0]\n\n    def call_git_success(self, args, files=None, expect_stderr=False):\n        \"\"\"Call git and return true if the call exit code of 0.\n\n        Parameters\n        ----------\n        args : list of str\n          Arguments to pass to `git`.\n        files : list of str, optional\n          File arguments to pass to `git`. The advantage of passing these here\n          rather than as part of `args` is that the call will be split into\n          multiple calls to avoid exceeding the maximum command line length.\n        expect_stderr : bool, optional\n          Standard error is expected and should not be elevated above the DEBUG\n          level.\n\n        Returns\n        -------\n        bool\n        \"\"\"\n        try:\n            self._git_custom_command(files, [\"git\"] + args,\n                                     expect_fail=True,\n                                     expect_stderr=expect_stderr,\n                                     check_fake_dates=True)\n        except CommandError:\n            return False\n        return True\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Register remote pointing to a url\n        \"\"\"\n        cmd = ['git', 'remote', 'add']\n        if options:\n            cmd += options\n        cmd += [name, url]\n\n        result = self._git_custom_command('', cmd)\n        self.config.reload()\n        return result\n\n    def remove_remote(self, name):\n        \"\"\"Remove existing remote\n        \"\"\"\n\n        # TODO: testing and error handling!\n        from .exceptions import RemoteNotAvailableError\n        try:\n            out, err = self._git_custom_command(\n                '', ['git', 'remote', 'remove', name])\n        except CommandError as e:\n            if 'fatal: No such remote' in e.stderr:\n                raise RemoteNotAvailableError(name,\n                                              cmd=\"git remote remove\",\n                                              msg=\"No such remote\",\n                                              stdout=e.stdout,\n                                              stderr=e.stderr)\n            else:\n                raise e\n\n        # TODO: config.reload necessary?\n        self.config.reload()\n        return\n\n    def update_remote(self, name=None, verbose=False):\n        \"\"\"\n        \"\"\"\n        options = [\"-v\"] if verbose else []\n        name = [name] if name else []\n        self._git_custom_command(\n            '', ['git', 'remote'] + name + ['update'] + options,\n            expect_stderr=True\n        )\n\n    def fetch(self, remote=None, refspec=None, all_=False, git_options=None,\n              **kwargs):\n        \"\"\"Fetches changes from a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to fetch from. If no remote is given and\n          `all_` is not set, the tracking branch is fetched.\n        refspec : str, optional\n          refspec to fetch.\n        all_ : bool, optional\n          fetch all remotes (and all of their branches).\n          Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-fetch.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-fetch.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        return list(\n            self.fetch_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    @guard_BadName\n    def fetch_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `fetch`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'fetch', '--verbose', '--progress'],\n            action='fetch',\n            urlvars=('remote.{}.url', 'remote.{}.url'),\n            protocol=GitProgress,\n            info_cls=FetchInfo,\n            info_from=1,\n            add_remote=False,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    # XXX Consider removing this method. It is only used in `update()`,\n    # where it could be easily replaced with fetch+merge\n    def pull(self, remote=None, refspec=None, git_options=None, **kwargs):\n        \"\"\"Pulls changes from a remote.\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to pull from. If no remote is given,\n          the remote tracking branch is used.\n        refspec : str, optional\n          refspec to fetch.\n        git_options : list, optional\n          Additional command line options for git-pull.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-pull.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n\n        cmd = ['git', 'pull', '--progress'] + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or fetch all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            # No explicit remote to fetch.\n            # => get tracking branch:\n            tb_remote, refspec = self.get_tracking_branch()\n            if tb_remote is not None:\n                remote = tb_remote\n            else:\n                # No remote, no tracking branch\n                # => fail\n                raise ValueError(\"Neither a remote is specified to pull \"\n                                 \"from nor a tracking branch is set up.\")\n\n        cmd.append(remote)\n        if refspec:\n            cmd += ensure_list(refspec)\n\n        # best effort to enable SSH connection caching\n        url = self.config.get('remote.{}.url'.format(remote), None)\n        if url and is_ssh(url):\n            ssh_manager.get_connection(url).open()\n        WitlessRunner(\n            cwd=self.path,\n            env=GitRunner.get_git_environ_adjusted()).run(\n                cmd,\n                protocol=StdOutCaptureWithGitProgress,\n        )\n\n    def push(self, remote=None, refspec=None, all_remotes=False,\n             all_=False, git_options=None, **kwargs):\n        \"\"\"Push changes to a remote (or all remotes).\n\n        Parameters\n        ----------\n        remote : str, optional\n          name of the remote to push to. If no remote is given and\n          `all_` is not set, the tracking branch is pushed.\n        refspec : str, optional\n          refspec to push.\n        all_ : bool, optional\n          push to all remotes. Fails if `remote` was given.\n        git_options : list, optional\n          Additional command line options for git-push.\n        kwargs :\n          Deprecated. GitPython-style keyword argument for git-push.\n          Will be appended to any git_options.\n        \"\"\"\n        git_options = ensure_list(git_options)\n        if kwargs:\n            git_options.extend(to_options(**kwargs))\n        if all_remotes:\n            # be nice to the elderly\n            all_ = True\n        return list(\n            self.push_(\n                remote=remote,\n                refspec=refspec,\n                all_=all_,\n                git_options=git_options,\n            )\n        )\n\n    def push_(self, remote=None, refspec=None, all_=False, git_options=None):\n        \"\"\"Like `push`, but returns a generator\"\"\"\n        yield from self._fetch_push_helper(\n            base_cmd=['git', 'push', '--progress', '--porcelain'],\n            action='push',\n            urlvars=('remote.{}.pushurl', 'remote.{}.url'),\n            protocol=StdOutCaptureWithGitProgress,\n            info_cls=PushInfo,\n            info_from=0,\n            add_remote=True,\n            remote=remote,\n            refspec=refspec,\n            all_=all_,\n            git_options=git_options)\n\n    def _fetch_push_helper(\n            self,\n            base_cmd,     # arg list\n            action,       # label fetch|push\n            urlvars,      # variables to query for URLs\n            protocol,     # processor for output\n            info_cls,     # Push|FetchInfo\n            info_from,    # 0=stdout, 1=stderr\n            add_remote,   # whether to add a 'remote' field to the info dict\n            remote=None, refspec=None, all_=False, git_options=None):\n\n        git_options = ensure_list(git_options)\n\n        cmd = base_cmd + git_options\n\n        if remote is None:\n            if refspec:\n                # conflicts with using tracking branch or push all remotes\n                # For now: Just fail.\n                # TODO: May be check whether it fits to tracking branch\n                raise ValueError(\n                    \"refspec specified without a remote. ({})\".format(refspec))\n            if all_:\n                remotes_to_process = self.get_remotes(with_urls_only=True)\n            else:\n                # No explicit remote to fetch.\n                # => get tracking branch:\n                tb_remote, refspec = self.get_tracking_branch()\n                if tb_remote is not None:\n                    remotes_to_process = [tb_remote]\n                else:\n                    # No remote, no tracking branch\n                    # => fail\n                    raise ValueError(\n                        \"Neither a remote is specified to {} \"\n                        \"from nor a tracking branch is set up.\".format(action))\n        else:\n            if all_:\n                raise ValueError(\n                    \"Option 'all_' conflicts with specified remote \"\n                    \"'{}'.\".format(remote))\n            remotes_to_process = [remote]\n\n        if refspec:\n            # prep for appending to cmd\n            refspec = ensure_list(refspec)\n\n        # no need for progress report, when there is just one remote\n        log_remote_progress = len(remotes_to_process) > 1\n        if log_remote_progress:\n            pbar_id = '{}remotes-{}'.format(action, id(self))\n            log_progress(\n                lgr.info,\n                pbar_id,\n                'Start %sing remotes for %s', action, self,\n                total=len(remotes_to_process),\n                label=action.capitalize(),\n                unit=' Remotes',\n            )\n        try:\n            for remote in remotes_to_process:\n                r_cmd = cmd + [remote]\n                if refspec:\n                    r_cmd += refspec\n\n                if log_remote_progress:\n                    log_progress(\n                        lgr.info,\n                        pbar_id,\n                        '{}ing remote %s'.format(action.capitalize()),\n                        remote,\n                        update=1,\n                        increment=True,\n                    )\n                # best effort to enable SSH connection caching\n                url = self.config.get(\n                    # make two attempts to get a URL\n                    urlvars[0].format(remote),\n                    self.config.get(\n                        urlvars[1].format(remote),\n                        None)\n                )\n                if url and is_ssh(url):\n                    ssh_manager.get_connection(url).open()\n                try:\n                    out = WitlessRunner(\n                        cwd=self.path,\n                        env=GitRunner.get_git_environ_adjusted()).run(\n                            r_cmd,\n                            protocol=protocol,\n                    )\n                    output = out[info_from] or ''\n                except CommandError as e:\n                    # intercept some errors that we express as an error report\n                    # in the info dicts\n                    if re.match('error: failed to (push|fetch) some refs', e.stderr):\n                        output = {1: e.stderr, 0: e.stdout}[info_from]\n                        if output is None:\n                            output = ''\n                    else:\n                        raise\n\n                for line in output.splitlines():\n                    try:\n                        # push info doesn't identify a remote, add it here\n                        pi = info_cls._from_line(line)\n                        if add_remote:\n                            pi['remote'] = remote\n                        yield pi\n                    except Exception:\n                        # it is not progress and no push info\n                        # don't hide it completely\n                        lgr.debug('git-%s reported: %s', action, line)\n        finally:\n            if log_remote_progress:\n                log_progress(\n                    lgr.info,\n                    pbar_id,\n                    'Finished %sing remotes for %s', action, self,\n                )\n\n    def get_remote_url(self, name, push=False):\n        \"\"\"Get the url of a remote.\n\n        Reads the configuration of remote `name` and returns its url or None,\n        if there is no url configured.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        push: bool\n          if True, get the pushurl instead of the fetch url.\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        return self.config.get(var, None)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL\n        \"\"\"\n\n        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')\n        self.config.set(var, url, where='local', reload=True)\n\n    def get_branch_commits_(self, branch=None, limit=None, stop=None):\n        \"\"\"Return commit hexshas for a branch\n\n        Parameters\n        ----------\n        branch: str, optional\n          If not provided, assumes current branch\n        limit: None | 'left-only', optional\n          Limit which commits to report.  If None -- all commits (merged or not),\n          if 'left-only' -- only the commits from the left side of the tree upon\n          merges\n        stop: str, optional\n          hexsha of the commit at which stop reporting (matched one is not\n          reported either)\n\n        Yields\n        ------\n        str\n        \"\"\"\n        cmd = ['rev-list']\n        if limit == 'left-only':\n            cmd.append('--left-only')\n        if not branch:\n            branch = self.get_active_branch()\n        cmd.append(branch)\n        for r in self.call_git_items_(cmd):\n            if stop and stop == r:\n                return\n            yield r\n\n    def checkout(self, name, options=None):\n        \"\"\"\n        \"\"\"\n        # TODO: May be check for the need of -b options herein?\n        cmd = ['git', 'checkout']\n        if options:\n            cmd += options\n        cmd += [str(name)]\n\n        self._git_custom_command('', cmd, expect_stderr=True, updates_tree=True)\n\n    # TODO: Before implementing annex merge, find usages and check for a needed\n    # change to call super().merge\n    def merge(self, name, options=None, msg=None, allow_unrelated=False, **kwargs):\n        if options is None:\n            options = []\n        if msg:\n            options = options + [\"-m\", msg]\n        if allow_unrelated and external_versions['cmd:git'] >= '2.9':\n            options += ['--allow-unrelated-histories']\n        self._git_custom_command(\n            '', ['git', 'merge'] + options + [name],\n            check_fake_dates=True,\n            **kwargs\n        )\n\n    def remove_branch(self, branch):\n        self._git_custom_command(\n            '', ['git', 'branch', '-D', branch]\n        )\n\n    def cherry_pick(self, commit):\n        \"\"\"Cherry pick `commit` to the current branch.\n\n        Parameters\n        ----------\n        commit : str\n            A single commit.\n        \"\"\"\n        self._git_custom_command(\"\", [\"git\", \"cherry-pick\", commit],\n                                 check_fake_dates=True)\n\n    @property\n    def dirty(self):\n        \"\"\"Is the repository dirty?\n\n        Note: This provides a quick answer when you simply want to know if\n        there are any untracked changes or modifications in this repository or\n        its submodules. For finer-grained control and more detailed reporting,\n        use status() instead.\n        \"\"\"\n        stdout, _ = self._git_custom_command(\n            [],\n            [\"git\", \"status\", \"--porcelain\",\n             # Ensure the result isn't influenced by status.showUntrackedFiles.\n             \"--untracked-files=normal\",\n             # Ensure the result isn't influenced by diff.ignoreSubmodules.\n             \"--ignore-submodules=none\"])\n        return bool(stdout.strip())\n\n    @property\n    def untracked_files(self):\n        \"\"\"Legacy interface, do not use! Use the status() method instead.\n\n        Despite its name, it also reports on untracked datasets, and\n        yields their names with trailing path separators.\n        \"\"\"\n        return [\n            '{}{}'.format(\n                str(p.relative_to(self.pathobj)),\n                os.sep if props['type'] != 'file' else ''\n            )\n            for p, props in self.status(\n                    untracked='all', eval_submodule_state='no').items()\n            if props.get('state', None) == 'untracked'\n        ]\n\n    def gc(self, allow_background=False, auto=False):\n        \"\"\"Perform house keeping (garbage collection, repacking)\"\"\"\n        cmd_options = ['git']\n        if not allow_background:\n            cmd_options += ['-c', 'gc.autodetach=0']\n        cmd_options += ['gc', '--aggressive']\n        if auto:\n            cmd_options += ['--auto']\n        self._git_custom_command('', cmd_options)\n\n    def _parse_gitmodules(self):\n        # TODO read .gitconfig from Git blob?\n        gitmodules = self.pathobj / '.gitmodules'\n        if not gitmodules.exists():\n            return {}\n        # pull out file content\n        out, err = self._git_custom_command(\n            '',\n            ['git', 'config', '-z', '-l', '--file', '.gitmodules'])\n        # abuse our config parser\n        db, _ = _parse_gitconfig_dump(out, {}, None, True, cwd=self.path)\n        mods = {}\n        for k, v in db.items():\n            if not k.startswith('submodule.'):\n                # we don't know what this is\n                lgr.warning(\"Skip unrecognized .gitmodule specification: %s=%s\", k, v)\n                continue\n            k_l = k.split('.')\n            # module name is everything after 'submodule.' that is not the variable\n            # name\n            mod_name = '.'.join(k_l[1:-1])\n            mod = mods.get(mod_name, {})\n            # variable name is the last 'dot-free' segment in the key\n            mod[k_l[-1]] = v\n            mods[mod_name] = mod\n\n        out = {}\n        # bring into traditional shape\n        for name, props in mods.items():\n            if 'path' not in props:\n                lgr.warning(\"Failed to get '%s.path', skipping this submodule\", name)\n                continue\n            modprops = {'gitmodule_{}'.format(k): v\n                        for k, v in props.items()\n                        if not (k.startswith('__') or k == 'path')}\n            modpath = self.pathobj / PurePosixPath(props['path'])\n            modprops['gitmodule_name'] = name\n            out[modpath] = modprops\n        return out\n\n    def get_submodules_(self, paths=None):\n        \"\"\"Yield submodules in this repository.\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n\n        Returns\n        -------\n        A generator that yields a dictionary with information for each\n        submodule.\n        \"\"\"\n        if not (self.pathobj / \".gitmodules\").exists():\n            return\n\n        modinfo = self._parse_gitmodules()\n        for path, props in self.get_content_info(\n                paths=paths,\n                ref=None,\n                untracked='no',\n                eval_file_type=False).items():\n            if props.get('type', None) != 'dataset':\n                continue\n            props[\"path\"] = path\n            props.update(modinfo.get(path, {}))\n            yield props\n\n    def get_submodules(self, sorted_=True, paths=None, compat=True):\n        \"\"\"Return list of submodules.\n\n        Parameters\n        ----------\n        sorted_ : bool, optional\n            Sort submodules by path name.\n        paths : list(pathlib.PurePath), optional\n            Restrict submodules to those under `paths`.\n        compat : bool, optional\n            If true, return a namedtuple that incompletely mimics the\n            attributes of GitPython's Submodule object in hope of backwards\n            compatibility with previous callers. Note that this form should be\n            considered temporary and callers should be updated; this flag will\n            be removed in a future release.\n\n        Returns\n        -------\n        List of submodule namedtuples if `compat` is true or otherwise a list\n        of dictionaries as returned by `get_submodules_`.\n        \"\"\"\n        xs = self.get_submodules_(paths=paths)\n        if compat:\n            warnings.warn(\"The attribute-based return value of get_submodules() \"\n                          \"exists for compatibility purposes and will be removed \"\n                          \"in an upcoming release\",\n                          DeprecationWarning)\n            xs = (Submodule(name=p[\"gitmodule_name\"],\n                            path=str(p[\"path\"].relative_to(self.pathobj)),\n                            url=p[\"gitmodule_url\"])\n                  for p in xs)\n\n        if sorted_:\n            if compat:\n                def key(x):\n                    return x.path\n            else:\n                def key(x):\n                    return x[\"path\"]\n            xs = sorted(xs, key=key)\n        return list(xs)\n\n    def add_submodule(self, path, name=None, url=None, branch=None):\n        \"\"\"Add a new submodule to the repository.\n\n        This will alter the index as well as the .gitmodules file, but will not\n        create a new commit.  If the submodule already exists, no matter if the\n        configuration differs from the one provided, the existing submodule\n        is considered as already added and no further action is performed.\n\n        Parameters\n        ----------\n        path : str\n          repository-relative path at which the submodule should be located, and\n          which will be created as required during the repository initialization.\n        name : str or None\n          name/identifier for the submodule. If `None`, the `path` will be used\n          as name.\n        url : str or None\n          git-clone compatible URL. If `None`, the repository is assumed to\n          exist, and the url of the first remote is taken instead. This is\n          useful if you want to make an existing repository a submodule of\n          another one.\n        branch : str or None\n          name of branch to be checked out in the submodule. The given branch\n          must exist in the remote repository, and will be checked out locally\n          as a tracking branch. If `None`, remote HEAD will be checked out.\n        \"\"\"\n        if name is None:\n            name = Path(path).as_posix()\n        # XXX the following should do it, but GitPython will refuse to add a submodule\n        # unless you specify a URL that is configured as one of its remotes, or you\n        # specify no URL, but the repo has at least one remote.\n        # this is stupid, as for us it is valid to not have any remote, because we can\n        # still obtain the submodule from a future publication location, based on the\n        # parent\n        # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch)\n        # going git native instead\n        cmd = ['submodule', 'add', '--name', name]\n        if branch is not None:\n            cmd += ['-b', branch]\n        if url is None:\n            # repo must already exist locally\n            subm = GitRepo(op.join(self.path, path), create=False, init=False)\n            # check that it has a commit, and refuse\n            # to operate on it otherwise, or we would get a bastard\n            # submodule that cripples git operations\n            if not subm.get_hexsha():\n                raise InvalidGitRepositoryError(\n                    'cannot add subdataset {} with no commits'.format(subm))\n            # make an attempt to configure a submodule source URL based on the\n            # discovered remote configuration\n            remote, branch = subm.get_tracking_branch()\n            url = subm.get_remote_url(remote) if remote else None\n\n        if url is None:\n            # had no luck with a remote URL\n            if not isabs(path):\n                # need to recode into a relative path \"URL\" in POSIX\n                # style, even on windows\n                url = posixpath.join(curdir, posix_relpath(path))\n            else:\n                url = path\n        cmd += [url, Path(path).as_posix()]\n        self.call_git(cmd)\n        # record dataset ID if possible for comprehesive metadata on\n        # dataset components within the dataset itself\n        subm_id = GitRepo(op.join(self.path, path)).config.get(\n            'datalad.dataset.id', None)\n        if subm_id:\n            self.call_git(\n                ['config', '--file', '.gitmodules', '--replace-all',\n                 'submodule.{}.datalad-id'.format(name), subm_id])\n        # ensure supported setup\n        _fixup_submodule_dotgit_setup(self, path)\n        # TODO: return value\n\n    def deinit_submodule(self, path, **kwargs):\n        \"\"\"Deinit a submodule\n\n        Parameters\n        ----------\n        path: str\n            path to the submodule; relative to `self.path`\n        kwargs:\n            see `__init__`\n        \"\"\"\n\n        self._git_custom_command(path,\n                                 ['git', 'submodule', 'deinit'] +\n                                 to_options(**kwargs))\n        # TODO: return value\n\n    def update_submodule(self, path, mode='checkout', init=False):\n        \"\"\"Update a registered submodule.\n\n        This will make the submodule match what the superproject expects by\n        cloning missing submodules and updating the working tree of the\n        submodules. The \"updating\" can be done in several ways depending\n        on the value of submodule.<name>.update configuration variable, or\n        the `mode` argument.\n\n        Parameters\n        ----------\n        path : str\n          Identifies which submodule to operate on by it's repository-relative\n          path.\n        mode : {checkout, rebase, merge}\n          Update procedure to perform. 'checkout': the commit recorded in the\n          superproject will be checked out in the submodule on a detached HEAD;\n          'rebase': the current branch of the submodule will be rebased onto\n          the commit recorded in the superproject; 'merge': the commit recorded\n          in the superproject will be merged into the current branch in the\n          submodule.\n        init : bool\n          If True, initialize all submodules for which \"git submodule init\" has\n          not been called so far before updating.\n          Primarily provided for internal purposes and should not be used directly\n          since would result in not so annex-friendly .git symlinks/references\n          instead of full featured .git/ directories in the submodules\n        \"\"\"\n        if GitRepo.is_valid_repo(self.pathobj / path):\n            subrepo = GitRepo(self.pathobj / path, create=False)\n            subbranch = subrepo.get_active_branch() if subrepo else None\n            subbranch_hexsha = subrepo.get_hexsha(subbranch) if subrepo else None\n        else:\n            subrepo = None\n            subbranch = None\n            subbranch_hexsha = None\n\n        cmd = ['git', 'submodule', 'update', '--%s' % mode]\n        if init:\n            cmd.append('--init')\n            subgitpath = opj(self.path, path, '.git')\n            if not exists(subgitpath):\n                # TODO:  wouldn't with --init we get all those symlink'ed .git/?\n                # At least let's warn\n                lgr.warning(\n                    \"Do not use update_submodule with init=True to avoid git creating \"\n                    \"symlinked .git/ directories in submodules\"\n                )\n            #  yoh: I thought I saw one recently but thought it was some kind of\n            #  an artifact from running submodule update --init manually at\n            #  some point, but looking at this code now I worry that it was not\n        self._git_custom_command(path, cmd)\n\n        if not init:\n            return\n\n        # track branch originally cloned, only if we had a valid repo at the start\n        updated_subbranch = subrepo.get_active_branch() if subrepo else None\n        if subbranch and not updated_subbranch:\n            # got into 'detached' mode\n            # trace if current state is a predecessor of the branch_hexsha\n            lgr.debug(\n                \"Detected detached HEAD after updating submodule %s which was \"\n                \"in %s branch before\", self.path, subbranch)\n            detached_hexsha = subrepo.get_hexsha()\n            if subrepo.get_merge_base(\n                    [subbranch_hexsha, detached_hexsha]) == detached_hexsha:\n                # TODO: config option?\n                # in all likely event it is of the same branch since\n                # it is an ancestor -- so we could update that original branch\n                # to point to the state desired by the submodule, and update\n                # HEAD to point to that location\n                lgr.info(\n                    \"Submodule HEAD got detached. Resetting branch %s to point \"\n                    \"to %s. Original location was %s\",\n                    subbranch, detached_hexsha[:8], subbranch_hexsha[:8]\n                )\n                branch_ref = 'refs/heads/%s' % subbranch\n                subrepo.update_ref(branch_ref, detached_hexsha)\n                assert(subrepo.get_hexsha(subbranch) == detached_hexsha)\n                subrepo.update_ref('HEAD', branch_ref, symbolic=True)\n                assert(subrepo.get_active_branch() == subbranch)\n            else:\n                lgr.warning(\n                    \"%s has a detached HEAD since cloned branch %s has another common ancestor with %s\",\n                    subrepo.path, subbranch, detached_hexsha[:8]\n                )\n        # TODO: return value\n\n    def update_ref(self, ref, value, symbolic=False):\n        \"\"\"Update the object name stored in a ref \"safely\".\n\n        Just a shim for `git update-ref` call if not symbolic, and\n        `git symbolic-ref` if symbolic\n\n        Parameters\n        ----------\n        ref : str\n          Reference, such as `ref/heads/BRANCHNAME` or HEAD.\n        value : str\n          Value to update to, e.g. hexsha of a commit when updating for a\n          branch ref, or branch ref if updating HEAD\n        symbolic : None\n          To instruct if ref is symbolic, e.g. should be used in case of\n          ref=HEAD\n        \"\"\"\n        self._git_custom_command(\n            '',\n            ['git', 'symbolic-ref' if symbolic else 'update-ref', ref, value]\n        )\n\n    def tag(self, tag, message=None, commit=None, options=None):\n        \"\"\"Tag a commit\n\n        Parameters\n        ----------\n        tag : str\n          Custom tag label. Must be a valid tag name.\n        message : str, optional\n          If provided, adds ['-m', <message>] to the list of `git tag`\n          arguments.\n        commit : str, optional\n          If provided, will be appended as last argument to the `git tag` call,\n          and can be used to identify the commit that shall be tagged, if\n          not HEAD.\n        options : list, optional\n          Additional command options, inserted prior a potential `commit`\n          argument.\n        \"\"\"\n        # TODO: call in save.py complains about extensive logging. When does it\n        # happen in what way? Figure out, whether to just silence it or raise or\n        # whatever else.\n        args = ['tag']\n        if message:\n            args += ['-m', message]\n        if options is not None:\n            args.extend(options)\n        args.append(tag)\n        if commit:\n            args.append(commit)\n        self.call_git(args)\n\n    def get_tags(self, output=None):\n        \"\"\"Get list of tags\n\n        Parameters\n        ----------\n        output : str, optional\n          If given, limit the return value to a list of values matching that\n          particular key of the tag properties.\n\n        Returns\n        -------\n        list\n          Each item is a dictionary with information on a tag. At present\n          this includes 'hexsha', and 'name', where the latter is the string\n          label of the tag, and the former the hexsha of the object the tag\n          is attached to. The list is sorted by the creator date (committer\n          date for lightweight tags and tagger date for annotated tags), with\n          the most recent commit being the last element.\n        \"\"\"\n        tags = [\n            dict(\n                name=t['refname:strip=2'],\n                hexsha=t['object'] if t['object'] else t['objectname'],\n            )\n            for t in self.for_each_ref_(\n                fields=['refname:strip=2', 'objectname', 'object'],\n                pattern='refs/tags',\n                sort='creatordate')\n        ]\n        if output:\n            return [t[output] for t in tags]\n        else:\n            return tags\n\n    def describe(self, commitish=None, **kwargs):\n        \"\"\" Quick and dirty implementation to call git-describe\n\n        Parameters\n        ----------\n        kwargs:\n            transformed to cmdline options for git-describe;\n            see __init__ for description of the transformation\n        \"\"\"\n        # TODO: be more precise what failure to expect when and raise actual\n        # errors\n        cmd = ['git', 'describe'] + to_options(**kwargs)\n        if commitish is not None:\n            cmd.append(commitish)\n        try:\n            describe, outerr = self._git_custom_command(\n                [],\n                cmd,\n                expect_fail=True)\n            return describe.strip()\n        # TODO: WTF \"catch everything\"?\n        except:\n            return None\n\n    def get_tracking_branch(self, branch=None, remote_only=False):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        Parameters\n        ----------\n        branch: str\n            local branch to look up. If none is given, active branch is used.\n        remote_only : bool\n            Don't return a value if the upstream remote is set to \".\" (meaning\n            this repository).\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n        if branch is None:\n            branch = self.get_active_branch()\n            if branch is None:\n                return None, None\n\n        track_remote = self.config.get('branch.{0}.remote'.format(branch), None)\n        if remote_only and track_remote == \".\":\n            return None, None\n        track_branch = self.config.get('branch.{0}.merge'.format(branch), None)\n        return track_remote, track_branch\n\n    @property\n    def count_objects(self):\n        \"\"\"return dictionary with count, size(in KiB) information of git objects\n        \"\"\"\n\n        count_cmd = ['git', 'count-objects', '-v']\n        count_str, err = self._git_custom_command('', count_cmd)\n        count = {key: int(value)\n                 for key, value in [item.split(': ')\n                                    for item in count_str.split('\\n')\n                                    if len(item.split(': ')) == 2]}\n        return count\n\n    def get_git_attributes(self):\n        \"\"\"Query gitattributes which apply to top level directory\n\n        It is a thin compatibility/shortcut wrapper around more versatile\n        get_gitattributes which operates on a list of paths and returns\n        a dictionary per each path\n\n        Returns\n        -------\n        dict:\n          a dictionary with attribute name and value items relevant for the\n          top ('.') directory of the repository, and thus most likely the\n          default ones (if not overwritten with more rules) for all files within\n          repo.\n        \"\"\"\n        return self.get_gitattributes('.')['.']\n\n    def get_gitattributes(self, path, index_only=False):\n        \"\"\"Query gitattributes for one or more paths\n\n        Parameters\n        ----------\n        path: path or list\n          Path(s) to query. Paths may be relative or absolute.\n        index_only: bool\n          Flag whether to consider only gitattribute setting that are reflected\n          in the repository index, not just in the work tree content.\n\n        Returns\n        -------\n        dict:\n          Each key is a queried path (always relative to the repostiory root),\n          each value is a dictionary with attribute\n          name and value items. Attribute values are either True or False,\n          for set and unset attributes, or are the literal attribute value.\n        \"\"\"\n        path = ensure_list(path)\n        cmd = [\"git\", \"check-attr\", \"-z\", \"--all\"]\n        if index_only:\n            cmd.append('--cached')\n        stdout, stderr = self._git_custom_command(path, cmd)\n        # make sure we have one entry for each query path to\n        # simplify work with the result\n        attributes = {_normalize_path(self.path, p): {} for p in path}\n        attr = []\n        for item in stdout.split('\\0'):\n            attr.append(item)\n            if len(attr) < 3:\n                continue\n            # we have a full record\n            p, name, value = attr\n            attrs = attributes[p]\n            attrs[name] = \\\n                True if value == 'set' else False if value == 'unset' else value\n            # done, reset item\n            attr = []\n        return attributes\n\n    def set_gitattributes(self, attrs, attrfile='.gitattributes', mode='a'):\n        \"\"\"Set gitattributes\n\n        By default appends additional lines to `attrfile`. Note, that later\n        lines in `attrfile` overrule earlier ones, which may or may not be\n        what you want. Set `mode` to 'w' to replace the entire file by\n        what you provided in `attrs`.\n\n        Parameters\n        ----------\n        attrs : list\n          Each item is a 2-tuple, where the first element is a path pattern,\n          and the second element is a dictionary with attribute key/value\n          pairs. The attribute dictionary must use the same semantics as those\n          returned by `get_gitattributes()`. Path patterns can use absolute paths,\n          in which case they will be normalized relative to the directory\n          that contains the target .gitattributes file (see `attrfile`).\n        attrfile: path\n          Path relative to the repository root of the .gitattributes file the\n          attributes shall be set in.\n        mode: str\n          'a' to append .gitattributes, 'w' to replace it\n        \"\"\"\n\n        git_attributes_file = op.join(self.path, attrfile)\n        attrdir = op.dirname(git_attributes_file)\n        if not op.exists(attrdir):\n            os.makedirs(attrdir)\n        with open(git_attributes_file, mode) as f:\n            for pattern, attr in sorted(attrs, key=lambda x: x[0]):\n                # normalize the pattern relative to the target .gitattributes file\n                npath = _normalize_path(\n                    op.join(self.path, op.dirname(attrfile)), pattern)\n                # paths in gitattributes always have to be POSIX\n                npath = Path(npath).as_posix()\n                attrline = u''\n                if npath.count(' '):\n                    # quote patterns with spaces\n                    attrline += u'\"{}\"'.format(npath.replace('\"', '\\\\\"'))\n                else:\n                    attrline += npath\n                for a in sorted(attr):\n                    val = attr[a]\n                    if val is True:\n                        attrline += ' {}'.format(a)\n                    elif val is False:\n                        attrline += ' -{}'.format(a)\n                    else:\n                        attrline += ' {}={}'.format(a, val)\n                f.write('\\n{}'.format(attrline))\n\n    def get_content_info(self, paths=None, ref=None, untracked='all',\n                         eval_file_type=True):\n        \"\"\"Get identifier and type information from repository content.\n\n        This is simplified front-end for `git ls-files/tree`.\n\n        Both commands differ in their behavior when queried about subdataset\n        paths. ls-files will not report anything, ls-tree will report on the\n        subdataset record. This function uniformly follows the behavior of\n        ls-tree (report on the respective subdataset mount).\n\n        Parameters\n        ----------\n        paths : list(pathlib.PurePath)\n          Specific paths, relative to the resolved repository root, to query\n          info for. Paths must be normed to match the reporting done by Git,\n          i.e. no parent dir components (ala \"some/../this\").\n          If none are given, info is reported for all content.\n        ref : gitref or None\n          If given, content information is retrieved for this Git reference\n          (via ls-tree), otherwise content information is produced for the\n          present work tree (via ls-files). With a given reference, the\n          reported content properties also contain a 'bytesize' record,\n          stating the size of a file in bytes.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when no `ref` was given:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_file_type : bool\n          If True, inspect file type of untracked files, and report annex\n          symlink pointers as type 'file'. This convenience comes with a\n          cost; disable to get faster performance if this information\n          is not needed.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n\n            Note that the reported type will not always match the type of\n            content committed to Git, rather it will reflect the nature\n            of the content minus platform/mode-specifics. For example,\n            a symlink to a locked annexed file on Unix will have a type\n            'file', reported, while a symlink to a file in Git or directory\n            will be of type 'symlink'.\n\n          `gitshasum`\n            SHASUM of the item as tracked by Git, or None, if not\n            tracked. This could be different from the SHASUM of the file\n            in the worktree, if it was modified.\n\n        Raises\n        ------\n        ValueError\n          In case of an invalid Git reference (e.g. 'HEAD' in an empty\n          repository)\n        \"\"\"\n        lgr.debug('%s.get_content_info(...)', self)\n        # TODO limit by file type to replace code in subdatasets command\n        info = OrderedDict()\n\n        if paths:\n            # path matching will happen against what Git reports\n            # and Git always reports POSIX paths\n            # any incoming path has to be relative already, so we can simply\n            # convert unconditionally\n            paths = [ut.PurePosixPath(p) for p in paths]\n\n        path_strs = list(map(str, paths)) if paths else None\n\n        # this will not work in direct mode, but everything else should be\n        # just fine\n        if not ref:\n            # make sure no operations are pending before we figure things\n            # out in the worktree\n            self.precommit()\n\n            # --exclude-standard will make sure to honor and standard way\n            # git can be instructed to ignore content, and will prevent\n            # crap from contaminating untracked file reports\n            cmd = ['git', 'ls-files',\n                   '--stage', '-z', '-d', '-m', '--exclude-standard']\n            # untracked report mode, using labels from `git diff` option style\n            if untracked == 'all':\n                cmd.append('-o')\n            elif untracked == 'normal':\n                cmd += ['-o', '--directory', '--no-empty-directory']\n            elif untracked == 'no':\n                pass\n            else:\n                raise ValueError(\n                    'unknown value for `untracked`: {}'.format(untracked))\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) (?P<sha>.*) (.*)\\t(?P<fname>.*)$')\n\n            if path_strs:\n                # we need to get their within repo elements since ls-tree\n                # for paths within submodules returns nothing!\n                # see https://public-inbox.org/git/20190703193305.GF21553@hopa.kiewit.dartmouth.edu/T/#u\n                submodules = [s.path for s in self.get_submodules()]\n                path_strs = get_parent_paths(path_strs, submodules)\n        else:\n            cmd = ['git', 'ls-tree', ref, '-z', '-r', '--full-tree', '-l']\n            props_re = re.compile(\n                r'(?P<type>[0-9]+) ([a-z]*) (?P<sha>[^ ]*) [\\s]*(?P<size>[0-9-]+)\\t(?P<fname>.*)$')\n\n        lgr.debug('Query repo: %s', cmd)\n        try:\n            stdout, stderr = self._git_custom_command(\n                path_strs,\n                cmd,\n                log_stderr=True,\n                log_stdout=True,\n                # not sure why exactly, but log_online has to be false!\n                log_online=False,\n                expect_stderr=False,\n                shell=False,\n                # we don't want it to scream on stdout\n                expect_fail=True)\n        except CommandError as exc:\n            if \"fatal: Not a valid object name\" in str(exc):\n                raise InvalidGitReferenceError(ref)\n            raise\n        lgr.debug('Done query repo: %s', cmd)\n\n        if not eval_file_type:\n            _get_link_target = None\n        elif ref:\n            def _read_symlink_target_from_catfile(lines):\n                # it is always the second line, all checks done upfront\n                header = lines.readline()\n                if header.rstrip().endswith('missing'):\n                    # something we do not know about, should not happen\n                    # in real use, but guard against to avoid stalling\n                    return ''\n                return lines.readline().rstrip()\n\n            _get_link_target = BatchedCommand(\n                ['git', 'cat-file', '--batch'],\n                path=self.path,\n                output_proc=_read_symlink_target_from_catfile,\n            )\n        else:\n            def try_readlink(path):\n                try:\n                    return os.readlink(path)\n                except OSError:\n                    # readlink will fail if the symlink reported by ls-files is\n                    # not in the working tree (it could be removed or\n                    # unlocked). Fall back to a slower method.\n                    return op.realpath(path)\n\n            _get_link_target = try_readlink\n\n        try:\n            self._get_content_info_line_helper(\n                ref,\n                info,\n                stdout.split('\\0'),\n                props_re,\n                _get_link_target)\n        finally:\n            if ref and _get_link_target:\n                # cancel batch process\n                _get_link_target.close()\n\n        lgr.debug('Done %s.get_content_info(...)', self)\n        return info\n\n    def _get_content_info_line_helper(self, ref, info, lines,\n                                      props_re, get_link_target):\n        \"\"\"Internal helper of get_content_info() to parse Git output\"\"\"\n        mode_type_map = {\n            '100644': 'file',\n            '100755': 'file',\n            '120000': 'symlink',\n            '160000': 'dataset',\n        }\n        for line in lines:\n            if not line:\n                continue\n            inf = {}\n            props = props_re.match(line)\n            if not props:\n                # not known to Git, but Git always reports POSIX\n                path = ut.PurePosixPath(line)\n                inf['gitshasum'] = None\n            else:\n                # again Git reports always in POSIX\n                path = ut.PurePosixPath(props.group('fname'))\n\n            # revisit the file props after this path has not been rejected\n            if props:\n                inf['gitshasum'] = props.group('sha')\n                inf['type'] = mode_type_map.get(\n                    props.group('type'), props.group('type'))\n                if get_link_target and inf['type'] == 'symlink' and \\\n                        ((ref is None and '.git/annex/objects' in \\\n                          ut.Path(\n                            get_link_target(str(self.pathobj / path))\n                          ).as_posix()) or \\\n                         (ref and \\\n                          '.git/annex/objects' in get_link_target(\n                              u'{}:{}'.format(\n                                  ref, str(path))))\n                        ):\n                    # report annex symlink pointers as file, their\n                    # symlink-nature is a technicality that is dependent\n                    # on the particular mode annex is in\n                    inf['type'] = 'file'\n\n                if ref and inf['type'] == 'file':\n                    inf['bytesize'] = int(props.group('size'))\n\n            # join item path with repo path to get a universally useful\n            # path representation with auto-conversion and tons of other\n            # stuff\n            path = self.pathobj.joinpath(path)\n            if 'type' not in inf:\n                # be nice and assign types for untracked content\n                inf['type'] = 'symlink' if path.is_symlink() \\\n                    else 'directory' if path.is_dir() else 'file'\n            info[path] = inf\n\n    def status(self, paths=None, untracked='all', eval_submodule_state='full'):\n        \"\"\"Simplified `git status` equivalent.\n\n        Parameters\n        ----------\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list. If a query path points\n          into a subdataset, a report is made on the subdataset record\n          within the queried dataset only (no recursion).\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        lgr.debug('Query status of %r for %s paths',\n                  self, len(paths) if paths else 'all')\n        return self.diffstatus(\n            fr='HEAD' if self.get_hexsha() else None,\n            to=None,\n            paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state)\n\n    def diff(self, fr, to, paths=None, untracked='all',\n             eval_submodule_state='full'):\n        \"\"\"Like status(), but reports changes between to arbitrary revisions\n\n        Parameters\n        ----------\n        fr : str or None\n          Revision specification (anything that Git understands). Passing\n          `None` considers anything in the target state as new.\n        to : str or None\n          Revision specification (anything that Git understands), or None\n          to compare to the state of the work tree.\n        paths : list or None\n          If given, limits the query to the specified paths. To query all\n          paths specify `None`, not an empty list.\n        untracked : {'no', 'normal', 'all'}\n          If and how untracked content is reported when `to` is None:\n          'no': no untracked files are reported; 'normal': untracked files\n          and entire untracked directories are reported as such; 'all': report\n          individual files even in fully untracked directories.\n        eval_submodule_state : {'full', 'commit', 'no'}\n          If 'full' (the default), the state of a submodule is evaluated by\n          considering all modifications, with the treatment of untracked files\n          determined by `untracked`. If 'commit', the modification check is\n          restricted to comparing the submodule's HEAD commit to the one\n          recorded in the superdataset. If 'no', the state of the subdataset is\n          not evaluated.\n\n        Returns\n        -------\n        dict\n          Each content item has an entry under a pathlib `Path` object instance\n          pointing to its absolute path inside the repository (this path is\n          guaranteed to be underneath `Repo.path`).\n          Each value is a dictionary with properties:\n\n          `type`\n            Can be 'file', 'symlink', 'dataset', 'directory'\n          `state`\n            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.\n        \"\"\"\n        return {k: v for k, v in self.diffstatus(\n            fr=fr, to=to, paths=paths,\n            untracked=untracked,\n            eval_submodule_state=eval_submodule_state).items()\n            if v.get('state', None) != 'clean'}\n\n    def diffstatus(self, fr, to, paths=None, untracked='all',\n                   eval_submodule_state='full', eval_file_type=True,\n                   _cache=None):\n        \"\"\"Like diff(), but reports the status of 'clean' content too\"\"\"\n        return self._diffstatus(\n            fr, to, paths, untracked, eval_submodule_state, eval_file_type,\n            _cache)\n\n    def _diffstatus(self, fr, to, paths, untracked, eval_state,\n                    eval_file_type, _cache):\n        \"\"\"Just like diffstatus(), but supports an additional evaluation\n        state 'global'. If given, it will return a single 'modified'\n        (vs. 'clean') state label for the entire repository, as soon as\n        it can.\"\"\"\n        def _get_cache_key(label, paths, ref, untracked=None):\n            return self.path, label, tuple(paths) if paths else None, \\\n                ref, untracked\n\n        if _cache is None:\n            _cache = {}\n\n        if paths:\n            # at this point we must normalize paths to the form that\n            # Git would report them, to easy matching later on\n            paths = [ut.Path(p) for p in paths]\n            paths = [\n                p.relative_to(self.pathobj) if p.is_absolute() else p\n                for p in paths\n            ]\n\n        # TODO report more info from get_content_info() calls in return\n        # value, those are cheap and possibly useful to a consumer\n        # we need (at most) three calls to git\n        if to is None:\n            # everything we know about the worktree, including os.stat\n            # for each file\n            key = _get_cache_key('ci', paths, None, untracked)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=None, untracked=untracked,\n                    eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we want Git to tell us what it considers modified and avoid\n            # reimplementing logic ourselves\n            key = _get_cache_key('mod', paths, None)\n            if key in _cache:\n                modified = _cache[key]\n            else:\n                modified = set(\n                    self.pathobj.joinpath(ut.PurePosixPath(p))\n                    for p in self._git_custom_command(\n                        # low-level code cannot handle pathobjs\n                        [str(p) for p in paths] if paths else None,\n                        ['git', 'ls-files', '-z', '-m'])[0].split('\\0')\n                    if p)\n                _cache[key] = modified\n        else:\n            key = _get_cache_key('ci', paths, to)\n            if key in _cache:\n                to_state = _cache[key]\n            else:\n                to_state = self.get_content_info(\n                    paths=paths, ref=to, eval_file_type=eval_file_type)\n                _cache[key] = to_state\n            # we do not need worktree modification detection in this case\n            modified = None\n        # origin state\n        key = _get_cache_key('ci', paths, fr)\n        if key in _cache:\n            from_state = _cache[key]\n        else:\n            if fr:\n                from_state = self.get_content_info(\n                    paths=paths, ref=fr, eval_file_type=eval_file_type)\n            else:\n                # no ref means from nothing\n                from_state = {}\n            _cache[key] = from_state\n\n        status = OrderedDict()\n        for f, to_state_r in to_state.items():\n            props = None\n            if f not in from_state:\n                # this is new, or rather not known to the previous state\n                props = dict(\n                    state='added' if to_state_r['gitshasum'] else 'untracked',\n                )\n                if 'type' in to_state_r:\n                    props['type'] = to_state_r['type']\n            elif to_state_r['gitshasum'] == from_state[f]['gitshasum'] and \\\n                    (modified is None or f not in modified):\n                if to_state_r['type'] != 'dataset':\n                    # no change in git record, and no change on disk\n                    props = dict(\n                        # at this point we know that the reported object ids\n                        # for this file are identical in the to and from\n                        # records.  If to is None, we're comparing to the\n                        # working tree and a deleted file will still have an\n                        # identical id, so we need to check whether the file is\n                        # gone before declaring it clean. This working tree\n                        # check is irrelevant and wrong if to is a ref.\n                        state='clean' if to is not None or (f.exists() or \\\n                              f.is_symlink()) else 'deleted',\n                        type=to_state_r['type'],\n                    )\n                else:\n                    # a dataset\n                    props = dict(type=to_state_r['type'])\n                    if to is not None:\n                        # we can only be confident without looking\n                        # at the worktree, if we compare to a recorded\n                        # state\n                        props['state'] = 'clean'\n                    else:\n                        # report the shasum that we know, for further\n                        # wrangling of subdatasets below\n                        props['gitshasum'] = to_state_r['gitshasum']\n                        props['prev_gitshasum'] = from_state[f]['gitshasum']\n            else:\n                # change in git record, or on disk\n                props = dict(\n                    # TODO we could have a new file that is already staged\n                    # but had subsequent modifications done to it that are\n                    # unstaged. Such file would presently show up as 'added'\n                    # ATM I think this is OK, but worth stating...\n                    state='modified' if f.exists() or \\\n                    f.is_symlink() else 'deleted',\n                    # TODO record before and after state for diff-like use\n                    # cases\n                    type=to_state_r['type'],\n                )\n            state = props.get('state', None)\n            if eval_state == 'global' and \\\n                    state not in ('clean', None):\n                # any modification means globally 'modified'\n                return 'modified'\n            if state in ('clean', 'added', 'modified'):\n                props['gitshasum'] = to_state_r['gitshasum']\n                if 'bytesize' in to_state_r:\n                    # if we got this cheap, report it\n                    props['bytesize'] = to_state_r['bytesize']\n                elif props['state'] == 'clean' and 'bytesize' in from_state[f]:\n                    # no change, we can take this old size info\n                    props['bytesize'] = from_state[f]['bytesize']\n            if state in ('clean', 'modified', 'deleted'):\n                props['prev_gitshasum'] = from_state[f]['gitshasum']\n            status[f] = props\n\n        for f, from_state_r in from_state.items():\n            if f not in to_state:\n                # we new this, but now it is gone and Git is not complaining\n                # about it being missing -> properly deleted and deletion\n                # stages\n                status[f] = dict(\n                    state='deleted',\n                    type=from_state_r['type'],\n                    # report the shasum to distinguish from a plainly vanished\n                    # file\n                    gitshasum=from_state_r['gitshasum'],\n                )\n                if eval_state == 'global':\n                    return 'modified'\n\n        if to is not None or eval_state == 'no':\n            # if we have `to` we are specifically comparing against\n            # a recorded state, and this function only attempts\n            # to label the state of a subdataset, not investigate\n            # specifically what the changes in subdatasets are\n            # this is done by a high-level command like rev-diff\n            # so the comparison within this repo and the present\n            # `state` label are all we need, and they are done already\n            if eval_state == 'global':\n                return 'clean'\n            else:\n                return status\n\n        # loop over all subdatasets and look for additional modifications\n        for f, st in status.items():\n            f = str(f)\n            if 'state' in st or not st['type'] == 'dataset':\n                # no business here\n                continue\n            if not GitRepo.is_valid_repo(f):\n                # submodule is not present, no chance for a conflict\n                st['state'] = 'clean'\n                continue\n            # we have to recurse into the dataset and get its status\n            subrepo = GitRepo(f)\n            subrepo_commit = subrepo.get_hexsha()\n            st['gitshasum'] = subrepo_commit\n            # subdataset records must be labeled clean up to this point\n            # test if current commit in subdataset deviates from what is\n            # recorded in the dataset\n            st['state'] = 'modified' \\\n                if st['prev_gitshasum'] != subrepo_commit \\\n                else 'clean'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n            if eval_state == 'commit':\n                continue\n            # the recorded commit did not change, so we need to make\n            # a more expensive traversal\n            st['state'] = subrepo._diffstatus(\n                # we can use 'HEAD' because we know that the commit\n                # did not change. using 'HEAD' will facilitate\n                # caching the result\n                fr='HEAD',\n                to=None,\n                paths=None,\n                untracked=untracked,\n                eval_state='global',\n                eval_file_type=False,\n                _cache=_cache) if st['state'] == 'clean' else 'modified'\n            if eval_state == 'global' and st['state'] == 'modified':\n                return 'modified'\n\n        if eval_state == 'global':\n            return 'clean'\n        else:\n            return status\n\n    def _save_pre(self, paths, _status, **kwargs):\n        # helper to get an actionable status report\n        if paths is not None and not paths and not _status:\n            return\n        if _status is None:\n            if 'untracked' not in kwargs:\n                kwargs['untracked'] = 'normal'\n            status = self.status(\n                paths=paths,\n                **{k: kwargs[k] for k in kwargs\n                   if k in ('untracked', 'eval_submodule_state')})\n        else:\n            # we want to be able to add items down the line\n            # make sure to detach from prev. owner\n            status = _status.copy()\n        status = OrderedDict(\n            (k, v) for k, v in status.items()\n            if v.get('state', None) != 'clean'\n        )\n        return status\n\n    def get_staged_paths(self):\n        \"\"\"Returns a list of any stage repository path(s)\n\n        This is a rather fast call, as it will not depend on what is going on\n        in the worktree.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None,\n                ['git', 'diff', '--name-only', '--staged'],\n                cwd=self.path,\n                log_stderr=True,\n                log_stdout=True,\n                log_online=False,\n                expect_stderr=False,\n                expect_fail=True)\n        except CommandError as e:\n            lgr.debug(exc_str(e))\n            stdout = ''\n        return [f for f in stdout.split('\\n') if f]\n\n    def _save_post(self, message, status, partial_commit):\n        # helper to commit changes reported in status\n\n        # TODO remove pathobj stringification when commit() can\n        # handle it\n        to_commit = [str(f.relative_to(self.pathobj))\n                     for f, props in status.items()] \\\n                    if partial_commit else None\n        if not partial_commit or to_commit:\n            # we directly call GitRepo.commit() to avoid a whole slew\n            # if direct-mode safeguards and workarounds in the AnnexRepo\n            # implementation (which also run an additional dry-run commit\n            GitRepo.commit(\n                self,\n                files=to_commit,\n                msg=message,\n                options=None,\n                # do not raise on empty commit\n                # it could be that the `add` in this save-cycle has already\n                # brought back a 'modified' file into a clean state\n                careless=True,\n            )\n\n    def save(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Save dataset content.\n\n        Parameters\n        ----------\n        message : str or None\n          A message to accompany the changeset in the log. If None,\n          a default message is used.\n        paths : list or None\n          Any content with path matching any of the paths given in this\n          list will be saved. Matching will be performed against the\n          dataset status (GitRepo.status()), or a custom status provided\n          via `_status`. If no paths are provided, ALL non-clean paths\n          present in the repo status or `_status` will be saved.\n        _status : dict or None\n          If None, Repo.status() will be queried for the given `ds`. If\n          a dict is given, its content will be used as a constraint.\n          For example, to save only modified content, but no untracked\n          content, set `paths` to None and provide a `_status` that has\n          no entries for untracked content.\n        **kwargs :\n          Additional arguments that are passed to underlying Repo methods.\n          Supported:\n\n          - git : bool (passed to Repo.add()\n          - eval_submodule_state : {'full', 'commit', 'no'}\n            passed to Repo.status()\n          - untracked : {'no', 'normal', 'all'} - passed to Repo.satus()\n        \"\"\"\n        return list(\n            self.save_(\n                message=message,\n                paths=paths,\n                _status=_status,\n                **kwargs\n            )\n        )\n\n    def save_(self, message=None, paths=None, _status=None, **kwargs):\n        \"\"\"Like `save()` but working as a generator.\"\"\"\n        from datalad.interface.results import get_status_dict\n\n        status = self._save_pre(paths, _status, **kwargs)\n        if not status:\n            # all clean, nothing todo\n            lgr.debug('Nothing to save in %r, exiting early', self)\n            return\n\n        # three things are to be done:\n        # - remove (deleted if not already staged)\n        # - add (modified/untracked)\n        # - commit (with all paths that have been touched, to bypass\n        #   potential pre-staged bits)\n\n        need_partial_commit = True if self.get_staged_paths() else False\n\n        # remove first, because removal of a subds would cause a\n        # modification of .gitmodules to be added to the todo list\n        to_remove = [\n            # TODO remove pathobj stringification when delete() can\n            # handle it\n            str(f.relative_to(self.pathobj))\n            for f, props in status.items()\n            if props.get('state', None) == 'deleted' and\n            # staged deletions have a gitshasum reported for them\n            # those should not be processed as git rm will error\n            # due to them being properly gone already\n            not props.get('gitshasum', None)]\n        vanished_subds = any(\n            props.get('type', None) == 'dataset' and\n            props.get('state', None) == 'deleted'\n            for f, props in status.items())\n        if to_remove:\n            for r in self.remove(\n                    to_remove,\n                    # we would always see individual files\n                    recursive=False):\n                # TODO normalize result\n                yield get_status_dict(\n                    action='delete',\n                    refds=self.pathobj,\n                    # TODO make remove() report the type\n                    # for now it claims to report on files only\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r)),\n                    # make remove() report on failures too\n                    status='ok',\n                    logger=lgr)\n\n        # TODO this additonal query should not be, base on status as given\n        # if anyhow possible, however, when paths are given, status may\n        # not contain all required information. In case of path=None AND\n        # _status=None, we should be able to avoid this, because\n        # status should have the full info already\n        # looks for contained repositories\n        added_submodule = False\n        untracked_dirs = [f.relative_to(self.pathobj)\n                          for f, props in status.items()\n                          if props.get('state', None) == 'untracked' and\n                          props.get('type', None) == 'directory']\n        to_add_submodules = []\n        if untracked_dirs:\n            to_add_submodules = [sm for sm, sm_props in\n                self.get_content_info(\n                    untracked_dirs,\n                    ref=None,\n                    # request exhaustive list, so that everything that is\n                    # still reported as a directory must be its own repository\n                    untracked='all').items()\n                if sm_props.get('type', None) == 'directory']\n            to_add_submodules = _prune_deeper_repos(to_add_submodules)\n            for cand_sm in to_add_submodules:\n                branch = self.get_active_branch()\n                adjusted_match = ADJUSTED_BRANCH_EXPR.match(\n                    branch if branch else '')\n                try:\n                    self.add_submodule(\n                        str(cand_sm.relative_to(self.pathobj)),\n                        url=None,\n                        name=None,\n                        branch=adjusted_match.group('name') if adjusted_match\n                        else branch\n                    )\n                except (CommandError, InvalidGitRepositoryError) as e:\n                    yield get_status_dict(\n                        action='add_submodule',\n                        ds=self,\n                        path=self.pathobj / ut.PurePosixPath(cand_sm),\n                        status='error',\n                        message=e.stderr if hasattr(e, 'stderr')\n                        else ('not a Git repository: %s', exc_str(e)),\n                        logger=lgr)\n                    continue\n                # This mirrors the result structure yielded for\n                # to_stage_submodules below.\n                yield get_status_dict(\n                    action='add',\n                    refds=self.pathobj,\n                    type='file',\n                    key=None,\n                    path=self.pathobj / ut.PurePosixPath(cand_sm),\n                    status='ok',\n                    logger=lgr)\n                added_submodule = True\n        if not need_partial_commit:\n            # without a partial commit an AnnexRepo would ignore any submodule\n            # path in its add helper, hence `git add` them explicitly\n            to_stage_submodules = {\n                str(f.relative_to(self.pathobj)): props\n                for f, props in status.items()\n                if props.get('state', None) in ('modified', 'untracked')\n                and props.get('type', None) == 'dataset'}\n            if to_stage_submodules:\n                lgr.debug(\n                    '%i submodule path(s) to stage in %r %s',\n                    len(to_stage_submodules), self,\n                    to_stage_submodules\n                    if len(to_stage_submodules) < 10 else '')\n                for r in GitRepo._save_add(\n                        self,\n                        to_stage_submodules,\n                        git_opts=None):\n                    yield r\n\n        if added_submodule or vanished_subds:\n            # need to include .gitmodules in what needs saving\n            status[self.pathobj.joinpath('.gitmodules')] = dict(\n                type='file', state='modified')\n            if hasattr(self, 'annexstatus') and not kwargs.get('git', False):\n                # we cannot simply hook into the coming add-call\n                # as this would go to annex, so make a dedicted git-add\n                # call to ensure .gitmodules is not annexed\n                # in any normal DataLad dataset .gitattributes will\n                # prevent this, but in a plain repo it won't\n                # https://github.com/datalad/datalad/issues/3306\n                for r in GitRepo._save_add(\n                        self,\n                        {op.join(self.path, '.gitmodules'): None}):\n                    yield r\n        to_add = {\n            # TODO remove pathobj stringification when add() can\n            # handle it\n            str(f.relative_to(self.pathobj)): props\n            for f, props in status.items()\n            if (props.get('state', None) in ('modified', 'untracked') and\n                f not in to_add_submodules)}\n        if to_add:\n            lgr.debug(\n                '%i path(s) to add to %s %s',\n                len(to_add), self, to_add if len(to_add) < 10 else '')\n            for r in self._save_add(\n                    to_add,\n                    git_opts=None,\n                    **{k: kwargs[k] for k in kwargs\n                       if k in (('git',) if hasattr(self, 'annexstatus')\n                                else tuple())}):\n                yield r\n\n        self._save_post(message, status, need_partial_commit)\n        # TODO yield result for commit, prev helper checked hexsha pre\n        # and post...\n\n    def _save_add(self, files, git_opts=None):\n        \"\"\"Simple helper to add files in save()\"\"\"\n        from datalad.interface.results import get_status_dict\n        try:\n            # without --verbose git 2.9.3  add does not return anything\n            add_out = self._git_custom_command(\n                list(files.keys()),\n                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']\n            )\n            # get all the entries\n            for r in self._process_git_get_output(*add_out):\n                yield get_status_dict(\n                    action=r.get('command', 'add'),\n                    refds=self.pathobj,\n                    type='file',\n                    path=(self.pathobj / ut.PurePosixPath(r['file']))\n                    if 'file' in r else None,\n                    status='ok' if r.get('success', None) else 'error',\n                    key=r.get('key', None),\n                    # while there is no git-annex underneath here, we\n                    # tend to fake its behavior, so we can also support\n                    # this type of messaging\n                    message='\\n'.join(r['error-messages'])\n                    if 'error-messages' in r else None,\n                    logger=lgr)\n        except OSError as e:\n            lgr.error(\"add: %s\" % e)\n            raise\n\n\n# TODO\n# remove submodule: nope, this is just deinit_submodule + remove\n# status?\n\n\ndef _fixup_submodule_dotgit_setup(ds, relativepath):\n    \"\"\"Implementation of our current of .git in a subdataset\n\n    Each subdataset/module has its own .git directory where a standalone\n    repository would have it. No gitdir files, no symlinks.\n    \"\"\"\n    # move .git to superrepo's .git/modules, remove .git, create\n    # .git-file\n    path = opj(ds.path, relativepath)\n    subds_dotgit = opj(path, \".git\")\n\n    repo = GitRepo(path, create=False)\n    if repo.dot_git.parent == repo.pathobj:\n        # this is what we want\n        return\n\n    # first we want to remove any conflicting worktree setup\n    # done by git to find the checkout at the mountpoint of the\n    # submodule, if we keep that, any git command will fail\n    # after we move .git\n    # Ben: Shouldn't we re-setup a possible worktree afterwards?\n    repo.config.unset('core.worktree', where='local')\n    # what we have here is some kind of reference, remove and\n    # replace by the target\n    os.remove(subds_dotgit)\n    # make absolute\n    src_dotgit = str(repo.dot_git)\n    # move .git\n    from os import rename, listdir, rmdir\n    ensure_dir(subds_dotgit)\n    for dot_git_entry in listdir(src_dotgit):\n        rename(opj(src_dotgit, dot_git_entry),\n               opj(subds_dotgit, dot_git_entry))\n    assert not listdir(src_dotgit)\n    rmdir(src_dotgit)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `to_options` function rearranges command-line options, causing test mismatches

**Description:**
After updating the `to_options` function by importing code from GitPython, the order of generated command-line options has changed. This alteration leads to mismatches in tests that expect options in a specific sequence.

**Example Code:**
```python
def cmd_func(options=None):
    return ['git', 'annex', 'my_cmd'] + to_options(m='bla', force=True)

output = cmd_func()
print(output)
# Expected: ['git', 'annex', 'my_cmd', '--force', '-m', 'bla']
# Actual: ['git', 'annex', 'my_cmd', '-m', 'bla', '--force']
```

**Expected Behavior:**
The `to_options` function should generate command-line options in a consistent order, such as `--force` followed by `-m`, to match the expected command sequences in tests.

**Actual Behavior:**
The options are reordered, with `-m` appearing before `--force`, causing tests to fail due to the unexpected sequence.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/gitrepo.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to Git via GitPython

For further information on GitPython see http://gitpython.readthedocs.org/

"""

import re
import time
import os
import os.path as op
import warnings
from locale import getpreferredencoding


from itertools import chain
import logging
from collections import (
    OrderedDict,
    namedtuple
)
from os import linesep
from os.path import (
    join as opj,
    exists,
    normpath,
    isabs,
    commonprefix,
    relpath,
    realpath,
    dirname,
    basename,
    curdir,
    pardir,
    sep
)

import posixpath
from functools import wraps
from weakref import WeakValueDictionary

import git as gitpy
from gitdb.exc import BadName
from git.exc import (
    GitCommandError,
    NoSuchPathError,
    InvalidGitRepositoryError
)
from datalad.log import log_progress
from datalad.support.due import due, Doi

from datalad import ssh_manager
from datalad.cmd import (
    WitlessRunner,
    WitlessProtocol,
    GitRunner,
    BatchedCommand,
    run_gitcommand_on_file_list_chunks,
    StdOutErrCapture,
)
from datalad.config import (
    ConfigManager,
    _parse_gitconfig_dump
)

from datalad.consts import (
    ADJUSTED_BRANCH_EXPR,
)
from datalad.dochelpers import exc_str
import datalad.utils as ut
from datalad.utils import (
    Path,
    PurePosixPath,
    ensure_list,
    optional_args,
    on_windows,
    getpwd,
    posix_relpath,
    ensure_dir,
    generate_file_chunks,
    ensure_unicode,
    split_cmdline,
)

# imports from same module:
from .external_versions import external_versions
from .exceptions import (
    CommandError,
    DeprecatedError,
    FileNotInRepositoryError,
    GitIgnoreError,
    InvalidGitReferenceError,
    MissingBranchError,
    OutdatedExternalDependencyWarning,
    PathKnownToRepositoryError,
)
from .network import (
    RI,
    PathRI,
    is_ssh
)
from .path import get_parent_paths
from .repo import (
    PathBasedFlyweight,
    RepoInterface
)

#from git.objects.blob import Blob

# shortcuts
_curdirsep = curdir + sep
_pardirsep = pardir + sep


lgr = logging.getLogger('datalad.gitrepo')
_lgr_level = lgr.getEffectiveLevel()
if _lgr_level <= 2:
    from ..log import LoggerHelper
    # Let's also enable gitpy etc debugging
    gitpy_lgr = LoggerHelper(logtarget="git").get_initialized_logger()
    gitpy_lgr.setLevel(_lgr_level)
    gitpy_lgr.propagate = True

# Override default GitPython's DB backend to talk directly to git so it doesn't
# interfere with possible operations performed by gc/repack
default_git_odbt = gitpy.GitCmdObjectDB

# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used.
# Didn't work as expected on a first try. Probably there is a neatier way to
# log Exceptions from git commands.


def to_options(**kwargs):
    """Transform keyword arguments into a list of cmdline options

    Parameters
    ----------
    split_single_char_options: bool

    kwargs:

    Returns
    -------
    list
    """
    # TODO: borrow_docs!

    return gitpy.Git().transform_kwargs(**kwargs)


def _normalize_path(base_dir, path):
    """Helper to check paths passed to methods of this class.

    Checks whether `path` is beneath `base_dir` and normalizes it.
    Additionally paths are converted into relative paths with respect to
    `base_dir`, considering PWD in case of relative paths. This
    is intended to be used in repository classes, which means that
    `base_dir` usually will be the repository's base directory.

    Parameters
    ----------
    base_dir: str
        directory to serve as base to normalized, relative paths
    path: str
        path to be normalized

    Returns
    -------
    str:
        path, that is a relative path with respect to `base_dir`
    """
    if not path:
        return path

    base_dir = realpath(base_dir)  # realpath OK
    # path = normpath(path)
    # Note: disabled normpath, because it may break paths containing symlinks;
    # But we don't want to realpath relative paths, in case cwd isn't the
    # correct base.

    if isabs(path):
        # path might already be a symlink pointing to annex etc,
        # so realpath only its directory, to get "inline" with
        # realpath(base_dir) above
        path = opj(realpath(dirname(path)), basename(path))  # realpath OK
    # Executive decision was made to not do this kind of magic!
    #
    # elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:
    #     # If we are inside repository, rebuilt relative paths.
    #     path = opj(realpath(getpwd()), path)
    #
    # BUT with relative curdir/pardir start it would assume relative to curdir
    #
    elif path.startswith(_curdirsep) or path.startswith(_pardirsep):
        path = normpath(opj(realpath(getpwd()), path))  # realpath OK
    else:
        # We were called from outside the repo. Therefore relative paths
        # are interpreted as being relative to self.path already.
        return path

    if commonprefix([path, base_dir]) != base_dir:
        raise FileNotInRepositoryError(msg="Path outside repository: %s"
                                           % path, filename=path)

    return relpath(path, start=base_dir)


@optional_args
def normalize_path(func):
    """Decorator to provide unified path conversion for a single file

    Unlike normalize_paths, intended to be used for functions dealing with a
    single filename at a time

    Note
    ----
    This is intended to be used within the repository classes and therefore
    returns a class method!

    The decorated function is expected to take a path at
    first positional argument (after 'self'). Additionally the class `func`
    is a member of, is expected to have an attribute 'path'.
    """

    @wraps(func)
    def newfunc(self, file_, *args, **kwargs):
        file_new = _normalize_path(self.path, file_)
        return func(self, file_new, *args, **kwargs)

    return newfunc


@optional_args
def normalize_paths(func, match_return_type=True, map_filenames_back=False,
                    serialize=False):
    """Decorator to provide unified path conversions.

    Note
    ----
    This is intended to be used within the repository classes and therefore
    returns a class method!

    The decorated function is expected to take a path or a list of paths at
    first positional argument (after 'self'). Additionally the class `func`
    is a member of, is expected to have an attribute 'path'.

    Accepts either a list of paths or a single path in a str. Passes a list
    to decorated function either way, but would return based on the value of
    match_return_type and possibly input argument.

    If a call to the wrapped function includes normalize_path and it is False
    no normalization happens for that function call (used for calls to wrapped
    functions within wrapped functions, while possible CWD is within a
    repository)

    Parameters
    ----------
    match_return_type : bool, optional
      If True, and a single string was passed in, it would return the first
      element of the output (after verifying that it is a list of length 1).
      It makes easier to work with single files input.
    map_filenames_back : bool, optional
      If True and returned value is a dictionary, it assumes to carry entries
      one per file, and then filenames are mapped back to as provided from the
      normalized (from the root of the repo) paths
    serialize : bool, optional
      Loop through files giving only a single one to the function one at a time.
      This allows to simplify implementation and interface to annex commands
      which do not take multiple args in the same call (e.g. checkpresentkey)
    """

    @wraps(func)
    def newfunc(self, files, *args, **kwargs):

        normalize = _normalize_path if kwargs.pop('normalize_paths', True) \
            else lambda rpath, filepath: filepath

        if files:
            if isinstance(files, str) or not files:
                files_new = [normalize(self.path, files)]
                single_file = True
            elif isinstance(files, list):
                files_new = [normalize(self.path, path) for path in files]
                single_file = False
            else:
                raise ValueError("_files_decorator: Don't know how to handle "
                                 "instance of %s." % type(files))
        else:
            single_file = None
            files_new = []

        if map_filenames_back:
            def remap_filenames(out):
                """Helper to map files back to non-normalized paths"""
                if isinstance(out, dict):
                    assert(len(out) == len(files_new))
                    files_ = [files] if single_file else files
                    mapped = out.__class__()
                    for fin, fout in zip(files_, files_new):
                        mapped[fin] = out[fout]
                    return mapped
                else:
                    return out
        else:
            remap_filenames = lambda x: x

        if serialize:  # and not single_file:
            result = [
                func(self, f, *args, **kwargs)
                for f in files_new
            ]
        else:
            result = func(self, files_new, *args, **kwargs)

        if single_file is None:
            # no files were provided, nothing we can do really
            return result
        elif (result is None) or not match_return_type or not single_file:
            # If function doesn't return anything or no denormalization
            # was requested or it was not a single file
            return remap_filenames(result)
        elif single_file:
            if len(result) != 1:
                # Magic doesn't apply
                return remap_filenames(result)
            elif isinstance(result, (list, tuple)):
                return result[0]
            elif isinstance(result, dict) and tuple(result)[0] == files_new[0]:
                # assume that returned dictionary has files as keys.
                return tuple(result.values())[0]
            else:
                # no magic can apply
                return remap_filenames(result)
        else:
            return RuntimeError("should have not got here... check logic")

    return newfunc


def _remove_empty_items(list_):
    """Remove empty entries from list

    This is needed, since some functions of GitPython may convert
    an empty entry to '.', when used with a list of paths.

    Parameter:
    ----------
    list_: list of str

    Returns
    -------
    list of str
    """
    if not isinstance(list_, list):
        lgr.warning(
            "_remove_empty_items() called with non-list type: %s" % type(list_))
        return list_
    return [file_ for file_ in list_ if file_]


if "2.24.0" <= external_versions["cmd:git"] < "2.25.0":
    # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing
    # into untracked submodules when multiple pathspecs are given, returning
    # repositories that are deeper than the first level. This helper filters
    # these deeper levels out so that save_() doesn't fail trying to add them.
    #
    # This regression fixed with upstream's 072a231016 (2019-12-10).
    def _prune_deeper_repos(repos):
        firstlevel_repos = []
        prev = None
        for repo in sorted(repos):
            if not (prev and str(repo).startswith(prev)):
                prev = str(repo)
                firstlevel_repos.append(repo)
        return firstlevel_repos
else:
    def _prune_deeper_repos(repos):
        return repos


def Repo(*args, **kwargs):
    """Factory method around gitpy.Repo to consistently initiate with different
    backend
    """
    # TODO: This probably doesn't work as intended (or at least not as
    #       consistently as intended). gitpy.Repo could be instantiated by
    #       classmethods Repo.init or Repo.clone_from. In these cases 'odbt'
    #       would be needed as a parameter to these methods instead of the
    #       constructor.
    if 'odbt' not in kwargs:
        kwargs['odbt'] = default_git_odbt
    return gitpy.Repo(*args, **kwargs)


def guard_BadName(func):
    """A helper to guard against BadName exception

    Workaround for
    https://github.com/gitpython-developers/GitPython/issues/768
    also see https://github.com/datalad/datalad/issues/2550
    Let's try to precommit (to flush anything flushable) and do
    it again
    """

    @wraps(func)
    def wrapped(repo, *args, **kwargs):
        try:
            return func(repo, *args, **kwargs)
        except BadName:
            repo.precommit()
            return func(repo, *args, **kwargs)

    return wrapped


class GitProgress(WitlessProtocol):
    """Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """
    # inform super-class to capture stderr
    proc_err = True

    _num_op_codes = 10
    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \
        [1 << x for x in range(_num_op_codes)]
    STAGE_MASK = BEGIN | END
    OP_MASK = ~STAGE_MASK

    DONE_TOKEN = 'done.'
    TOKEN_SEPARATOR = ', '

    _known_ops = {
        COUNTING: ("Counting", "Objects"),
        ENUMERATING: ("Enumerating", "Objects"),
        COMPRESSING: ("Compressing", "Objects"),
        WRITING: ("Writing", "Objects"),
        RECEIVING: ("Receiving", "Objects"),
        RESOLVING: ("Resolving", "Deltas"),
        FINDING_SOURCES: ("Finding", "Sources"),
        CHECKING_OUT: ("Check out", "Things"),
    }

    __slots__ = ('_seen_ops', '_pbars', '_encoding')

    re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
    re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")

    def __init__(self, *args):
        super().__init__(*args)
        self._encoding = getpreferredencoding(do_setlocale=False)
        self._unprocessed = None

    def connection_made(self, transport):
        super().connection_made(transport)
        self._seen_ops = []
        self._pbars = set()

    def process_exited(self):
        # take down any progress bars that were not closed orderly
        for pbar_id in self._pbars:
            log_progress(
                lgr.info,
                pbar_id,
                'Finished',
            )
        super().process_exited()

    def pipe_data_received(self, fd, byts):
        # progress reports only come from stderr
        if fd != 2:
            # let the base class decide what to do with it
            super().pipe_data_received(fd, byts)
            return
        for line in byts.splitlines(keepends=True):
            # put any unprocessed content back in front
            line = self._unprocessed + line if self._unprocessed else line
            self._unprocessed = None
            if not self._parse_progress_line(line):
                # anything that doesn't look like a progress report
                # is retained and returned
                # in case of partial progress lines, this can lead to
                # leakage of progress info into the output, but
                # it is better to enable better (maybe more expensive)
                # subsequent filtering than hidding lines with
                # unknown, potentially important info
                lgr.debug('Non-progress stderr: %s', line)
                if line.endswith((b'\r', b'\n')):
                    # complete non-progress line, pass on
                    super().pipe_data_received(fd, line)
                else:
                    # an incomplete line, maybe the next batch completes
                    # it to become a recognizable progress report
                    self._unprocessed = line

    def _parse_progress_line(self, line):
        """Process a single line

        Parameters
        ----------
        line : bytes

        Returns
        -------
        bool
          Flag whether the line was recognized as a Git progress report.
        """
        # handle
        # Counting objects: 4, done.
        # Compressing objects:  50% (1/2)
        # Compressing objects: 100% (2/2)
        # Compressing objects: 100% (2/2), done.
        line = line.decode(self._encoding) if isinstance(line, bytes) else line
        if line.startswith(('warning:', 'error:', 'fatal:')):
            return False

        # find escape characters and cut them away - regex will not work with
        # them as they are non-ascii. As git might expect a tty, it will send them
        last_valid_index = None
        for i, c in enumerate(reversed(line)):
            if ord(c) < 32:
                # its a slice index
                last_valid_index = -i - 1
            # END character was non-ascii
        # END for each character in line
        if last_valid_index is not None:
            line = line[:last_valid_index]
        # END cut away invalid part
        line = line.rstrip()

        cur_count, max_count = None, None
        match = self.re_op_relative.match(line)
        if match is None:
            match = self.re_op_absolute.match(line)

        if not match:
            return False
        # END could not get match

        op_code = 0
        _remote, op_name, _percent, cur_count, max_count, message = match.groups()

        # get operation id
        if op_name == "Counting objects":
            op_code |= self.COUNTING
        elif op_name == "Compressing objects":
            op_code |= self.COMPRESSING
        elif op_name == "Writing objects":
            op_code |= self.WRITING
        elif op_name == 'Receiving objects':
            op_code |= self.RECEIVING
        elif op_name == 'Resolving deltas':
            op_code |= self.RESOLVING
        elif op_name == 'Finding sources':
            op_code |= self.FINDING_SOURCES
        elif op_name == 'Checking out files':
            op_code |= self.CHECKING_OUT
        elif op_name == 'Enumerating objects':
            op_code |= self.ENUMERATING
        else:
            # Note: On windows it can happen that partial lines are sent
            # Hence we get something like "CompreReceiving objects", which is
            # a blend of "Compressing objects" and "Receiving objects".
            # This can't really be prevented.
            lgr.debug(
                'Output line matched a progress report of an unknown type: %s',
                line)
            # TODO investigate if there is any chance that we might swallow
            # important info -- until them do not flag this line
            # as progress
            return False
        # END handle op code

        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)

        op_props = self._known_ops[op_code]

        # figure out stage
        if op_code not in self._seen_ops:
            self._seen_ops.append(op_code)
            op_code |= self.BEGIN
            log_progress(
                lgr.info,
                pbar_id,
                'Start {} {}'.format(
                    op_props[0].lower(),
                    op_props[1].lower(),
                ),
                label=op_props[0],
                unit=' {}'.format(op_props[1]),
                total=float(max_count) if max_count else None,
            )
            self._pbars.add(pbar_id)
        # END begin opcode

        if message is None:
            message = ''
        # END message handling

        done_progress = False
        message = message.strip()
        if message.endswith(self.DONE_TOKEN):
            op_code |= self.END
            message = message[:-len(self.DONE_TOKEN)]
            done_progress = True
        # END end message handling
        message = message.strip(self.TOKEN_SEPARATOR)

        if cur_count and max_count:
            log_progress(
                lgr.info,
                pbar_id,
                message,
                update=float(cur_count),
            )

        if done_progress:
            log_progress(
                lgr.info,
                pbar_id,
                'Finished {} {}'.format(
                    op_props[0].lower(),
                    op_props[1].lower(),
                ),
            )
            self._pbars.discard(pbar_id)
        return True


class StdOutCaptureWithGitProgress(GitProgress):
    proc_out = True


class FetchInfo(dict):
    """
    dict that carries results of a fetch operation of a single head

    Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """

    NEW_TAG, NEW_HEAD, HEAD_UPTODATE, TAG_UPDATE, REJECTED, FORCED_UPDATE, \
        FAST_FORWARD, ERROR = [1 << x for x in range(8)]

    _re_fetch_result = re.compile(r'^\s*(.) (\[?[\w\s\.$@]+\]?)\s+(.+) [-> ]+ ([^\s]+)(    \(.*\)?$)?')

    _flag_map = {
        '!': ERROR,
        '+': FORCED_UPDATE,
        '*': 0,
        '=': HEAD_UPTODATE,
        ' ': FAST_FORWARD,
        '-': TAG_UPDATE,
    }
    _operation_map = {
        NEW_TAG: 'new-tag',
        NEW_HEAD: 'new-branch',
        HEAD_UPTODATE: 'uptodate',
        TAG_UPDATE: 'tag-update',
        REJECTED: 'rejected',
        FORCED_UPDATE: 'forced-update',
        FAST_FORWARD: 'fast-forward',
        ERROR: 'error',
    }

    @classmethod
    def _from_line(cls, line):
        """Parse information from the given line as returned by git-fetch -v
        and return a new FetchInfo object representing this information.
        """
        match = cls._re_fetch_result.match(line)
        if match is None:
            raise ValueError("Failed to parse line: %r" % line)

        # parse lines
        control_character, operation, local_remote_ref, remote_local_ref, note = \
            match.groups()

        # parse flags from control_character
        flags = 0
        try:
            flags |= cls._flag_map[control_character]
        except KeyError:
            raise ValueError(
                "Control character %r unknown as parsed from line %r"
                % (control_character, line))
        # END control char exception handling

        # parse operation string for more info - makes no sense for symbolic refs,
        # but we parse it anyway
        old_commit = None
        if 'rejected' in operation:
            flags |= cls.REJECTED
        if 'new tag' in operation:
            flags |= cls.NEW_TAG
        if 'tag update' in operation:
            flags |= cls.TAG_UPDATE
        if 'new branch' in operation:
            flags |= cls.NEW_HEAD
        if '...' in operation or '..' in operation:
            split_token = '...'
            if control_character == ' ':
                split_token = split_token[:-1]
            old_commit = operation.split(split_token)[0]
        # END handle refspec

        return cls(
            ref=remote_local_ref.strip(),
            local_ref=local_remote_ref.strip(),
            # convert flag int into a list of operation labels
            operations=[
                cls._operation_map[o]
                for o in cls._operation_map.keys()
                if flags & o
            ],
            note=note,
            old_commit=old_commit,
        )


class PushInfo(dict):
    """dict that carries results of a push operation of a single head

    Reduced variant of GitPython's RemoteProgress class

    Original copyright:
        Copyright (C) 2008, 2009 Michael Trier and contributors
    Original license:
        BSD 3-Clause "New" or "Revised" License
    """
    NEW_TAG, NEW_HEAD, NO_MATCH, REJECTED, REMOTE_REJECTED, REMOTE_FAILURE, DELETED, \
        FORCED_UPDATE, FAST_FORWARD, UP_TO_DATE, ERROR = [1 << x for x in range(11)]

    _flag_map = {'X': NO_MATCH,
                 '-': DELETED,
                 '*': 0,
                 '+': FORCED_UPDATE,
                 ' ': FAST_FORWARD,
                 '=': UP_TO_DATE,
                 '!': ERROR}

    _operation_map = {
        NEW_TAG: 'new-tag',
        NEW_HEAD: 'new-branch',
        NO_MATCH: 'no-match',
        REJECTED: 'rejected',
        REMOTE_REJECTED: 'remote-rejected',
        REMOTE_FAILURE: 'remote-failure',
        DELETED: 'deleted',
        FORCED_UPDATE: 'forced-update',
        FAST_FORWARD: 'fast-forward',
        UP_TO_DATE: 'uptodate',
        ERROR: 'error',
    }

    @classmethod
    def _from_line(cls, line):
        """Create a new PushInfo instance as parsed from line which is expected to be like
            refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes"""
        control_character, from_to, summary = line.split('\t', 3)
        flags = 0

        # control character handling
        try:
            flags |= cls._flag_map[control_character]
        except KeyError:
            raise ValueError("Control character %r unknown as parsed from line %r" % (control_character, line))
        # END handle control character

        # from_to handling
        from_ref_string, to_ref_string = from_to.split(':')

        # commit handling, could be message or commit info
        old_commit = None
        if summary.startswith('['):
            if "[rejected]" in summary:
                flags |= cls.REJECTED
            elif "[remote rejected]" in summary:
                flags |= cls.REMOTE_REJECTED
            elif "[remote failure]" in summary:
                flags |= cls.REMOTE_FAILURE
            elif "[no match]" in summary:
                flags |= cls.ERROR
            elif "[new tag]" in summary:
                flags |= cls.NEW_TAG
            elif "[new branch]" in summary:
                flags |= cls.NEW_HEAD
            # uptodate encoded in control character
        else:
            # fast-forward or forced update - was encoded in control character,
            # but we parse the old and new commit
            split_token = "..."
            if control_character == " ":
                split_token = ".."
            old_sha, _new_sha = summary.split(' ')[0].split(split_token)
            # have to use constructor here as the sha usually is abbreviated
            old_commit = old_sha
        # END message handling

        return cls(
            from_ref=from_ref_string.strip(),
            to_ref=to_ref_string.strip(),
            # convert flag int into a list of operation labels
            operations=[
                cls._operation_map[o]
                for o in cls._operation_map.keys()
                if flags & o
            ],
            note=summary.strip(),
            old_commit=old_commit,
        )


# Compatibility kludge.  See GitRepo.get_submodules().
Submodule = namedtuple("Submodule", ["name", "path", "url"])


class GitRepo(RepoInterface, metaclass=PathBasedFlyweight):
    """Representation of a git repository

    """
    # We must check git config to have name and email set, but
    # should do it once
    _config_checked = False

    # Begin Flyweight:

    _unique_instances = WeakValueDictionary()

    def _flyweight_invalid(self):
        return not self.is_valid_git()

    @classmethod
    def _flyweight_reject(cls, id_, *args, **kwargs):
        # TODO:
        # This is a temporary approach. See PR # ...
        # create = kwargs.pop('create', None)
        # kwargs.pop('path', None)
        # if create and kwargs:
        #     # we have `create` plus options other than `path`
        #     return "Call to {0}() with args {1} and kwargs {2} conflicts " \
        #            "with existing instance {3}." \
        #            "This is likely to be caused by inconsistent logic in " \
        #            "your code." \
        #            "".format(cls, args, kwargs, cls._unique_instances[id_])
        pass

    # End Flyweight

    def __hash__(self):
        # the flyweight key is already determining unique instances
        # add the class name to distinguish from strings of a path
        return hash((self.__class__.__name__, self.__weakref__.key))

    # This is the least common denominator to claim that a user
    # used DataLad.
    # For now citing Zenodo's all (i.e., latest) version
    @due.dcite(Doi("10.5281/zenodo.808846"),
               # override path since there is no need ATM for such details
               path="datalad",
               description="DataLad - Data management and distribution platform")
    def __init__(self, path, url=None, runner=None, create=True,
                 git_opts=None, repo=None, fake_dates=False,
                 create_sanity_checks=True,
                 **kwargs):
        """Creates representation of git repository at `path`.

        Can also be used to create a git repository at `path`.

        Parameters
        ----------
        path: str
          path to the git repository; In case it's not an absolute path,
          it's relative to PWD
        url: str, optional
          DEPRECATED -- use .clone() class method
          url to the to-be-cloned repository. Requires a valid git url
          according to:
          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .
        create: bool, optional
          if true, creates a git repository at `path` if there is none. Also
          creates `path`, if it doesn't exist.
          If set to false, an exception is raised in case `path` doesn't exist
          or doesn't contain a git repository.
        repo: git.Repo, optional
          GitPython's Repo instance to (re)use if provided
        create_sanity_checks: bool, optional
          Whether to perform sanity checks during initialization (when
          `create=True` and target path is not a valid repo already), such as
          that new repository is not created in the directory where git already
          tracks some files.
        kwargs:
          keyword arguments serving as additional options to the git-init
          command. Therefore, it makes sense only if called with `create`.

          Generally, this way of passing options to the git executable is
          (or will be) used a lot in this class. It's a transformation of
          python-style keyword arguments (or a `dict`) to command line arguments,
          provided by GitPython.

          A single character keyword will be prefixed by '-', multiple characters
          by '--'. An underscore in the keyword becomes a dash. The value of the
          keyword argument is used as the value for the corresponding command
          line argument. Assigning a boolean creates a flag.

          Examples:
          no_commit=True => --no-commit
          C='/my/path'   => -C /my/path

        """

        # BEGIN Repo validity test
        # We want to fail early for tests, that would be performed a lot. In particular this is about
        # GitRepo.is_valid_repo. We would use the latter to decide whether or not to call GitRepo() only for
        # __init__ to then test the same things again. If we fail early we can save the additional test from outer
        # scope.
        self.path = path

        # Note, that the following three path objects are used often and therefore
        # are stored for performance. Path object creation comes with a cost. Most noteably,
        # this is used for validity checking of the repository.
        self.pathobj = ut.Path(self.path)
        self.dot_git = self.pathobj / '.git'
        # Read a potential .git file in order to not do that over and over again, when testing is_valid_git() etc.
        # TODO: There's still some code duplication with static method GitRepo.get_git_dir()
        #       However, it's returning relative path. So, the logic in usage needs to be unified in order to melt both
        #       pieces.
        if self.dot_git.is_file():
            with self.dot_git.open() as f:
                line = f.readline()
                if line.startswith("gitdir: "):
                    self.dot_git = self.pathobj / line[7:].strip()
                else:
                    raise InvalidGitRepositoryError("Invalid .git file")

        elif self.dot_git.is_symlink():
            self.dot_git = self.dot_git.resolve()

        self._valid_git_test_path = self.dot_git / 'HEAD'
        _valid_repo = self.is_valid_git()

        do_create = False
        if create and not _valid_repo:
            if repo is not None:
                # `repo` passed with `create`, which doesn't make sense
                raise TypeError("argument 'repo' must not be used with 'create'")
            do_create = True
        else:
            # Note: We used to call gitpy.Repo(path) here, which potentially
            # raised NoSuchPathError or InvalidGitRepositoryError. This is
            # used by callers of GitRepo.__init__() to detect whether we have a
            # valid repo at `path`. Now, with switching to lazy loading property
            # `repo`, we detect those cases without instantiating a
            # gitpy.Repo().

            if not exists(path):
                raise NoSuchPathError(path)
            if not _valid_repo:
                raise InvalidGitRepositoryError(path)
        # END Repo validity test

        if url is not None:
            raise DeprecatedError(
                new=".clone() class method",
                version="0.5.0",
                msg="RF: url passed to init()"
            )

        # So that we "share" control paths with git/git-annex
        if ssh_manager:
            ssh_manager.assure_initialized()

        # note: we may also want to distinguish between a path to the worktree
        # and the actual repository

        # Could be used to e.g. disable automatic garbage and autopacking
        # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0']
        self._GIT_COMMON_OPTIONS = []
        # actually no need with default GitPython db backend not in memory
        # default_git_odbt but still allows for faster testing etc.
        # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = []

        if git_opts is None:
            git_opts = {}
        if kwargs:
            git_opts.update(kwargs)

        self.cmd_call_wrapper = runner or GitRunner(cwd=self.path)
        self._repo = repo
        self._cfg = None

        if do_create:  # we figured it out ealier
            self._repo = self._create_empty_repo(path,
                                                 create_sanity_checks, **git_opts)

        # inject git options into GitPython's git call wrapper:
        # Note: `None` currently can happen, when Runner's protocol prevents
        # calls above from being actually executed (DryRunProtocol)
        if self._repo is not None:
            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS

        # with DryRunProtocol path might still not exist
        if exists(self.path):
            self.inode = os.stat(self.path).st_ino
        else:
            self.inode = None

        if fake_dates:
            self.configure_fake_dates()
        # Set by fake_dates_enabled to cache config value across this instance.
        self._fake_dates_enabled = None

    def _create_empty_repo(self, path, sanity_checks=True, **kwargs):
        if not op.lexists(path):
            os.makedirs(path)
        elif sanity_checks and external_versions['cmd:git'] < '2.14.0':
            warnings.warn(
                "Your git version (%s) is too old, we will not safe-guard "
                "against creating a new repository under already known to git "
                "subdirectory" % external_versions['cmd:git'],
                OutdatedExternalDependencyWarning
            )
        elif sanity_checks:
            # Verify that we are not trying to initialize a new git repository
            # under a directory some files of which are already tracked by git
            # use case: https://github.com/datalad/datalad/issues/3068
            try:
                stdout, _ = self._git_custom_command(
                    None, ['git', 'ls-files'], cwd=path, expect_fail=True
                )
                if stdout:
                    raise PathKnownToRepositoryError(
                        "Failing to initialize new repository under %s where "
                        "following files are known to a repository above: %s"
                        % (path, stdout)
                    )
            except CommandError:
                # assume that all is good -- we are not under any repo
                pass

        cmd = ['git', 'init']
        cmd.extend(kwargs.pop('_from_cmdline_', []))
        cmd.extend(to_options(**kwargs))
        lgr.debug(
            "Initialize empty Git repository at '%s'%s",
            path,
            ' %s' % cmd[2:] if cmd[2:] else '')

        try:
            stdout, stderr = self._git_custom_command(
                None,
                cmd,
                cwd=path,
                log_stderr=True,
                log_stdout=True,
                log_online=False,
                expect_stderr=False,
                shell=False,
                # we don't want it to scream on stdout
                expect_fail=True)
        except CommandError as exc:
            lgr.error(exc_str(exc))
            raise
        # we want to return None and have lazy eval take care of
        # the rest
        return

    @property
    def repo(self):
        # with DryRunProtocol path not exist
        if exists(self.path):
            inode = os.stat(self.path).st_ino
        else:
            inode = None
        if self.inode != inode:

            # TODO: - what if we never had any inode? possible?
            #       - also: call to self._repo and only afterwards checking whether it's None seems strange

            # reset background processes invoked by GitPython:
            # it can be that we never got to instantiate a GitPython repo instance
            if self._repo:
                self._repo.git.clear_cache()
            self.inode = inode

        if self._repo is None:
            # Note, that this may raise GitCommandError, NoSuchPathError,
            # InvalidGitRepositoryError:
            self._repo = self.cmd_call_wrapper(
                Repo,
                self.path)
            lgr.log(8, "Using existing Git repository at %s", self.path)

        # inject git options into GitPython's git call wrapper:
        # Note: `None` currently can happen, when Runner's protocol prevents
        # call of Repo(path) above from being actually executed (DryRunProtocol)
        if self._repo is not None:
            self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS

        return self._repo

    @classmethod
    def clone(cls, url, path, *args, clone_options=None, **kwargs):
        """Clone url into path

        Provides workarounds for known issues (e.g.
        https://github.com/datalad/datalad/issues/785)

        Parameters
        ----------
        url : str
        path : str
        clone_options : dict
          Key/value pairs of arbitrary options that will be passed on to the
          underlying call to `git-clone`.
        expect_fail : bool
          Whether expect that command might fail, so error should be logged then
          at DEBUG level instead of ERROR
        """

        if 'repo' in kwargs:
            raise TypeError("argument 'repo' conflicts with cloning")
            # TODO: what about 'create'?

        expect_fail = kwargs.pop('expect_fail', False)
        # fail early on non-empty target:
        from os import listdir
        if exists(path) and listdir(path):
            # simulate actual GitCommandError:
            lgr.warning("destination path '%s' already exists and is not an "
                        "empty directory." % path)
            raise GitCommandError(
                ['git', 'clone', '-v', url, path],
                128,
                "fatal: destination path '%s' already exists and is not an "
                "empty directory." % path)
        else:
            # protect against cloning into existing and obviously dangling
            # instance for that location
            try:
                del cls._unique_instances[path]
            except KeyError:
                # didn't exist - all fine
                pass

        # Massage URL
        url_ri = RI(url) if not isinstance(url, RI) else url
        if not on_windows:
            # if we are on windows, the local path of a URL
            # would not end up being a proper local path and cloning
            # would fail. Don't try to be smart and just pass the
            # URL along unmodified

            # try to get a local path from `url`:
            try:
                url = url_ri.localpath
                url_ri = RI(url)
            except ValueError:
                pass

        if is_ssh(url_ri):
            ssh_manager.get_connection(url).open()
        else:
            if isinstance(url_ri, PathRI):
                # expand user, because execution not going through a shell
                # doesn't work well otherwise
                new_url = os.path.expanduser(url)
                if url != new_url:
                    lgr.info("Expanded source path to %s from %s", new_url, url)
                    url = new_url

        fix_annex = None
        ntries = 5  # 3 is not enough for robust workaround
        for trial in range(ntries):
            try:
                lgr.debug("Git clone from {0} to {1}".format(url, path))

                out, err = WitlessRunner(
                    env=GitRunner.get_git_environ_adjusted()).run(
                        ['git', 'clone', '--progress', url, path] \
                        + (to_options(**clone_options)
                           if clone_options else []),
                        protocol=GitProgress,
                )
                # fish out non-critical warnings by git-clone
                # (empty repo clone, etc.), all other content is logged
                # by the progress helper to 'debug'
                for errline in err.splitlines():
                    if errline.startswith('warning:'):
                        lgr.warning(errline[8:].strip())
                lgr.debug("Git clone completed")
                break
            except CommandError as e:
                # log here but let caller decide what to do
                e_str = exc_str(e)
                # see https://github.com/datalad/datalad/issues/785
                if re.search("Request for .*aborted.*Unable to find", str(e),
                             re.DOTALL) \
                        and trial < ntries - 1:
                    lgr.info(
                        "Hit a known issue with Git (see GH#785). Trial #%d, "
                        "retrying",
                        trial)
                    continue
                    (lgr.debug if expect_fail else lgr.error)(e_str)

                if "Clone succeeded, but checkout failed." in str(e):
                    fix_annex = e
                    break

                raise

        # get ourselves a repository instance
        gr = cls(path, *args, **kwargs)
        if fix_annex:
            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then
            # would need to import our own subclass
            if hasattr(gr, 'is_valid_annex'):
                lgr.warning("Experienced issues while cloning. "
                            "Trying to fix it, using git-annex-fsck.")
                if not gr.is_initialized():
                    gr._init()
                gr.fsck()
            else:
                lgr.warning("Experienced issues while cloning: %s", exc_str(fix_annex))
        return gr

    def __del__(self):
        # unbind possibly bound ConfigManager, to prevent all kinds of weird
        # stalls etc
        self._cfg = None
        # Make sure to flush pending changes, especially close batch processes
        # (internal `git cat-file --batch` by GitPython)
        try:
            if getattr(self, '_repo', None) is not None and exists(self.path):
                # gc might be late, so the (temporary)
                # repo doesn't exist on FS anymore
                self._repo.git.clear_cache()
                # We used to write out the index to flush GitPython's
                # state... but such unconditional write is really a workaround
                # and does not play nice with read-only operations - permission
                # denied etc. So disabled 
                #if exists(opj(self.path, '.git')):  # don't try to write otherwise
                #    self.repo.index.write()
        except (InvalidGitRepositoryError, AttributeError):
            # might have being removed and no longer valid or attributes unbound
            pass

    def __repr__(self):
        return "<GitRepo path=%s (%s)>" % (self.path, type(self))

    def __eq__(self, obj):
        """Decides whether or not two instances of this class are equal.

        This is done by comparing the base repository path.
        """
        return self.path == obj.path

    def is_valid_git(self):
        """Returns whether the underlying repository appears to be still valid

        Note, that this almost identical to the classmethod is_valid_repo(). However,
        if we are testing an existing instance, we can save Path object creations. Since this testing
        is done a lot, this is relevant. Creation of the Path objects in is_valid_repo() takes nearly half the time of
        the entire function.

        Also note, that this method is bound to an instance but still class-dependent, meaning that a subclass
        cannot simply overwrite it. This is particularly important for the call from within __init__(),
        which in turn is called by the subclasses' __init__. Using an overwrite would lead to the wrong thing being
        called.
        """

        return self.dot_git.exists() and (
                not self.dot_git.is_dir() or self._valid_git_test_path.exists()
        )

    @classmethod
    def is_valid_repo(cls, path):
        """Returns if a given path points to a git repository"""
        if not isinstance(path, Path):
            path = Path(path)
        dot_git_path = path / '.git'

        # the aim here is to have this test as cheap as possible, because
        # it is performed a lot
        # recognize two things as good-enough indicators of a present
        # repo: 1) a non-empty .git directory (#3473)
        #          NOTE: It's actually faster (and more accurate) to test for existence of a particular subpath.
        #                This should be something that's there right after git-init. Going for .git/HEAD ATM.
        #
        #                In [11]: %timeit path.exists() and (not path.is_dir() or head_path.exists())
        #                4.93 µs ± 34.8 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
        #                In [12]: %timeit path.exists() and (not path.is_dir() or any(path.iterdir()))
        #                12.8 µs ± 150 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
        #
        #   and 2) a pointer file or symlink

        return dot_git_path.exists() and (
            not dot_git_path.is_dir() or (dot_git_path / 'HEAD').exists()
        )

    @staticmethod
    def get_git_dir(repo):
        """figure out a repo's gitdir

        '.git' might be a  directory, a symlink or a file

        Note
        ----
        Please try using GitRepo.dot_git instead! That one's not static, but it's cheaper and you should avoid
        not having an instance of a repo you're working on anyway. Note, that the property in opposition to this method
        returns an absolute path.


        Parameters
        ----------
        repo: path or Repo instance
          currently expected to be the repos base dir

        Returns
        -------
        str
          relative path to the repo's git dir; So, default would be ".git"
        """
        if hasattr(repo, 'path'):
            # repo instance like given
            repo = repo.path
        dot_git = op.join(repo, ".git")
        if not op.exists(dot_git):
            raise RuntimeError("Missing .git in %s." % repo)
        elif op.islink(dot_git):
            git_dir = os.readlink(dot_git)
        elif op.isdir(dot_git):
            git_dir = ".git"
        elif op.isfile(dot_git):
            with open(dot_git) as f:
                git_dir = f.readline()
                if git_dir.startswith("gitdir:"):
                    git_dir = git_dir[7:]
                git_dir = git_dir.strip()

        return git_dir

    @property
    def config(self):
        """Get an instance of the parser for the persistent repository
        configuration.

        Note: This allows to also read/write .datalad/config,
        not just .git/config

        Returns
        -------
        ConfigManager
        """
        if self._cfg is None:
            # associate with this dataset and read the entire config hierarchy
            self._cfg = ConfigManager(dataset=self, source='any')
        return self._cfg

    def is_with_annex(self):
        """Report if GitRepo (assumed) has (remotes with) a git-annex branch
        """
        return any(
            b['refname:strip=2'] == 'git-annex' or b['refname:strip=2'].endswith('/git-annex')
            for b in self.for_each_ref_(fields='refname:strip=2', pattern=['refs/heads', 'refs/remotes'])
        )

    @classmethod
    def get_toppath(cls, path, follow_up=True, git_options=None):
        """Return top-level of a repository given the path.

        Parameters
        -----------
        follow_up : bool
          If path has symlinks -- they get resolved by git.  If follow_up is
          True, we will follow original path up until we hit the same resolved
          path.  If no such path found, resolved one would be returned.
        git_options: list of str
          options to be passed to the git rev-parse call

        Return None if no parent directory contains a git repository.
        """
        cmd = ['git']
        if git_options:
            cmd.extend(git_options)
        cmd += ["rev-parse", "--show-toplevel"]
        try:
            toppath, err = GitRunner().run(
                cmd,
                cwd=path,
                log_stdout=True, log_stderr=True,
                expect_fail=True, expect_stderr=True)
            toppath = toppath.rstrip('\n\r')
        except CommandError:
            return None
        except OSError:
            toppath = GitRepo.get_toppath(dirname(path), follow_up=follow_up,
                                          git_options=git_options)

        if follow_up:
            path_ = path
            path_prev = ""
            while path_ and path_ != path_prev:  # on top /.. = /
                if realpath(path_) == toppath:
                    toppath = path_
                    break
                path_prev = path_
                path_ = dirname(path_)

        return toppath

    @normalize_paths
    def add(self, files, git=True, git_options=None, update=False):
        """Adds file(s) to the repository.

        Parameters
        ----------
        files: list
          list of paths to add
        git: bool
          somewhat ugly construction to be compatible with AnnexRepo.add();
          has to be always true.
        update: bool
          --update option for git-add. From git's manpage:
           Update the index just where it already has an entry matching
           <pathspec>. This removes as well as modifies index entries to match
           the working tree, but adds no new files.

           If no <pathspec> is given when --update option is used, all tracked
           files in the entire working tree are updated (old versions of Git
           used to limit the update to the current directory and its
           subdirectories).

        Returns
        -------
        list
          Of status dicts.
        """
        # under all circumstances call this class' add_ (otherwise
        # AnnexRepo.add would go into a loop
        return list(GitRepo.add_(self, files, git=git, git_options=git_options,
                    update=update))

    def add_(self, files, git=True, git_options=None, update=False):
        """Like `add`, but returns a generator"""
        # TODO: git_options is used as options for the git-add here,
        # instead of options to the git executable => rename for consistency

        if not git:
            lgr.warning(
                'GitRepo.add() called with git=%s, this should not happen',
                git)
            git = True

        # there is no other way then to collect all files into a list
        # at this point, because we need to pass them at once to a single
        # `git add` call
        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]

        if not (files or git_options or update):
            # wondering why just a warning? in cmdline this is also not an error
            lgr.warning("add was called with empty file list and no options.")
            return

        try:
            # without --verbose git 2.9.3  add does not return anything
            add_out = self._git_custom_command(
                files,
                # Set annex.largefiles to prevent storing files in annex when
                # GitRepo() is instantiated with a v6+ annex repo.
                ['git', '-c', 'annex.largefiles=nothing', 'add'] +
                ensure_list(git_options) +
                to_options(update=update) + ['--verbose']
            )
            # get all the entries
            for o in self._process_git_get_output(*add_out):
                yield o
            # Note: as opposed to git cmdline, force is True by default in
            #       gitpython, which would lead to add things, that are
            #       ignored or excluded otherwise
            # 2. Note: There is an issue with globbing (like adding '.'),
            #       which apparently doesn't care for 'force' and therefore
            #       adds '.git/...'. May be it's expanded at the wrong
            #       point in time or sth. like that.
            # For now, use direct call to git add.
            #self.cmd_call_wrapper(self.repo.index.add, files, write=True,
            #                      force=False)
            # TODO: May be make use of 'fprogress'-option to indicate
            # progress
            # But then, we don't have it for git-annex add, anyway.
            #
            # TODO: Is write=True a reasonable way to do it?
            # May be should not write until success of operation is
            # confirmed?
            # What's best in case of a list of files?
        except OSError as e:
            lgr.error("add: %s" % e)
            raise

        # Make sure return value from GitRepo is consistent with AnnexRepo
        # currently simulating similar return value, assuming success
        # for all files:
        # TODO: Make return values consistent across both *Repo classes!
        return

    @staticmethod
    def _process_git_get_output(stdout, stderr=None):
        """Given both outputs (stderr is ignored atm) of git add - process it

        Primarily to centralize handling in both indirect annex and direct
        modes when ran through proxy
        """
        return [{u'file': f, u'success': True}
                for f in re.findall("'(.*)'[\n$]", ensure_unicode(stdout))]

    @normalize_paths(match_return_type=False)
    def remove(self, files, recursive=False, **kwargs):
        """Remove files.

        Calls git-rm.

        Parameters
        ----------
        files: str
          list of paths to remove
        recursive: False
          whether to allow recursive removal from subdirectories
        kwargs:
          see `__init__`

        Returns
        -------
        [str]
          list of successfully removed files.
        """

        files = _remove_empty_items(files)

        if recursive:
            kwargs['r'] = True
        stdout, stderr = self._git_custom_command(
            files, ['git', 'rm'] + to_options(**kwargs))

        # output per removed file is expected to be "rm 'PATH'":
        return [line.strip()[4:-1] for line in stdout.splitlines()]

        #return self.repo.git.rm(files, cached=False, **kwargs)

    def precommit(self):
        """Perform pre-commit maintenance tasks
        """
        # All GitPython commands should take care about flushing index
        # whenever they modify it, so we would not care to do anything
        # if self.repo is not None and exists(opj(self.path, '.git')):  # don't try to write otherwise:
        #     # flush possibly cached in GitPython changes to index:
        #     # if self.repo.git:
        #     #     sys.stderr.write("CLEARING\n")
        #     #     self.repo.git.clear_cache()
        #     self.repo.index.write()

        # Close batched by GitPython git processes etc
        # Ref: https://github.com/gitpython-developers/GitPython/issues/718
        self.repo.__del__()
        pass

    @staticmethod
    def _get_prefixed_commit_msg(msg):
        DATALAD_PREFIX = "[DATALAD]"
        return DATALAD_PREFIX if not msg else "%s %s" % (DATALAD_PREFIX, msg)

    def for_each_ref_(self, fields=('objectname', 'objecttype', 'refname'),
                      pattern=None, points_at=None, sort=None, count=None,
                      contains=None):
        """Wrapper for `git for-each-ref`

        Please see manual page git-for-each-ref(1) for a complete overview
        of its functionality. Only a subset of it is supported by this
        wrapper.

        Parameters
        ----------
        fields : iterable or str
          Used to compose a NULL-delimited specification for for-each-ref's
          --format option. The default field list reflects the standard
          behavior of for-each-ref when the --format option is not given.
        pattern : list or str, optional
          If provided, report only refs that match at least one of the given
          patterns.
        points_at : str, optional
          Only list refs which points at the given object.
        sort : list or str, optional
          Field name(s) to sort-by. If multiple fields are given, the last one
          becomes the primary key. Prefix any field name with '-' to sort in
          descending order.
        count : int, optional
          Stop iteration after the given number of matches.
        contains : str, optional
          Only list refs which contain the specified commit.

        Yields
        ------
        dict with items matching the given `fields`

        Raises
        ------
        ValueError
          if no `fields` are given

        RuntimeError
          if `git for-each-ref` returns a record where the number of
          properties does not match the number of `fields`
        """
        if not fields:
            raise ValueError('no `fields` provided, refuse to proceed')
        fields = ensure_list(fields)
        cmd = [
            "git",
            "for-each-ref",
            "--format={}".format(
                '%00'.join(
                    '%({})'.format(f) for f in fields)),
        ]
        if points_at:
            cmd.append('--points-at={}'.format(points_at))
        if contains:
            cmd.append('--contains={}'.format(contains))
        if sort:
            for k in ensure_list(sort):
                cmd.append('--sort={}'.format(k))
        if pattern:
            cmd += ensure_list(pattern)
        if count:
            cmd.append('--count={:d}'.format(count))

        out, _ = self._git_custom_command(None, cmd)
        for line in out.splitlines():
            props = line.split('\0')
            if len(fields) != len(props):
                raise RuntimeError(
                    'expected fields {} from git-for-each-ref, but got: {}'.format(
                        fields, props))
            yield dict(zip(fields, props))

    def configure_fake_dates(self):
        """Configure repository to use fake dates.
        """
        lgr.debug("Enabling fake dates")
        self.config.set("datalad.fake-dates", "true")

    @property
    def fake_dates_enabled(self):
        """Is the repository configured to use fake dates?
        """
        if self._fake_dates_enabled is None:
            self._fake_dates_enabled = \
                self.config.getbool('datalad', 'fake-dates', default=False)
        return self._fake_dates_enabled

    def add_fake_dates(self, env):
        """Add fake dates to `env`.

        Parameters
        ----------
        env : dict or None
            Environment variables.

        Returns
        -------
        A dict (copied from env), with date-related environment
        variables for git and git-annex set.
        """
        env = (env if env is not None else os.environ).copy()
        # Note: Use _git_custom_command here rather than repo.git.for_each_ref
        # so that we use annex-proxy in direct mode.
        last_date = list(self.for_each_ref_(
            fields='committerdate:raw',
            count=1,
            pattern='refs/heads',
            sort="-committerdate",
        ))

        if last_date:
            # Drop the "contextual" timezone, leaving the unix timestamp.  We
            # avoid :unix above because it wasn't introduced until Git v2.9.4.
            last_date = last_date[0]['committerdate:raw'].split()[0]
            seconds = int(last_date)
        else:
            seconds = self.config.obtain("datalad.fake-dates-start")
        seconds_new = seconds + 1
        date = "@{} +0000".format(seconds_new)

        lgr.debug("Setting date to %s",
                  time.strftime("%a %d %b %Y %H:%M:%S +0000",
                                time.gmtime(seconds_new)))

        env["GIT_AUTHOR_DATE"] = date
        env["GIT_COMMITTER_DATE"] = date
        env["GIT_ANNEX_VECTOR_CLOCK"] = str(seconds_new)

        return env

    def commit(self, msg=None, options=None, _datalad_msg=False, careless=True,
               files=None, date=None, index_file=None):
        """Commit changes to git.

        Parameters
        ----------
        msg: str, optional
          commit-message
        options: list of str, optional
          cmdline options for git-commit
        _datalad_msg: bool, optional
          To signal that commit is automated commit by datalad, so
          it would carry the [DATALAD] prefix
        careless: bool, optional
          if False, raise when there's nothing actually committed;
          if True, don't care
        files: list of str, optional
          path(s) to commit
        date: str, optional
          Date in one of the formats git understands
        index_file: str, optional
          An alternative index to use
        """

        self.precommit()

        # assemble commandline
        cmd = ['git', 'commit']
        options = ensure_list(options)

        if date:
            options += ["--date", date]

        if not msg:
            msg = 'Recorded changes'
            _datalad_msg = True

        if _datalad_msg:
            msg = self._get_prefixed_commit_msg(msg)

        options += ["-m", msg]
        cmd.extend(options)

        # set up env for commit
        env = GitRunner.get_git_environ_adjusted()
        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)
        if index_file:
            env['GIT_INDEX_FILE'] = index_file

        lgr.debug("Committing via direct call of git: %s" % cmd)

        file_chunks = generate_file_chunks(files, cmd) if files else [[]]

        runner = WitlessRunner(cwd=self.path, env=env)

        try:
            for i, chunk in enumerate(file_chunks):
                cur_cmd = cmd + (
                    # if this is an explicit dry-run, there is no point in
                    # amending, because no commit was ever made
                    # otherwise, amend the first commit, and prevent
                    # leaving multiple commits behind
                    ['--amend', '--no-edit']
                    if i > 0 and '--dry-run' not in cmd
                    else []
                ) + ['--'] + chunk
                runner.run(
                    cur_cmd,
                    protocol=StdOutErrCapture,
                    stdin=None,
                )
        except CommandError as e:
            # real errors first
            if "did not match any file(s) known to git" in e.stderr:
                raise FileNotInRepositoryError(
                    cmd=e.cmd,
                    msg="File(s) unknown to git",
                    code=e.code,
                    filename=linesep.join([
                        l for l in e.stderr.splitlines()
                        if l.startswith("error: pathspec")
                    ])
                )
            # behavior choices now
            elif not careless:
                # not willing to compromise at all
                raise
            elif 'nothing to commit' in e.stdout:
                lgr.debug("nothing to commit in %s. Ignored.", self)
            elif 'no changes added to commit' in e.stdout or \
                    'nothing added to commit' in e.stdout:
                lgr.debug("no changes added to commit in %s. Ignored.", self)
            else:
                raise

    # TODO usage is primarily in the tests, consider making a test helper and
    # remove from GitRepo API
    def get_indexed_files(self):
        """Get a list of files in git's index

        Returns
        -------
        list
            list of paths rooting in git's base dir
        """

        return [
            str(r.relative_to(self.pathobj))
            for r in self.get_content_info(
                paths=None, ref=None, untracked='no', eval_file_type=False)
        ]

    def format_commit(self, fmt, commitish=None):
        """Return `git show` output for `commitish`.

        Parameters
        ----------
        fmt : str
            A format string accepted by `git show`.
        commitish: str, optional
          Any commit identifier (defaults to "HEAD").

        Returns
        -------
        str or, if there are not commits yet, None.
        """
        cmd = ['git', 'show', '-z', '--no-patch', '--format=' + fmt]
        if commitish is not None:
            cmd.append(commitish + "^{commit}")
        # make sure Git takes our argument as a revision
        cmd.append('--')
        try:
            stdout, stderr = self._git_custom_command(
                '', cmd, expect_stderr=True, expect_fail=True)
        except CommandError as e:
            if 'bad revision' in e.stderr:
                raise ValueError("Unknown commit identifier: %s" % commitish)
            elif 'does not have any commits yet' in e.stderr:
                return None
            else:
                raise e
        # This trailing null is coming from the -z above, which avoids the
        # newline that Git would append to the output. We could drop -z and
        # strip the newline directly, but then we'd have to worry about
        # compatibility across platforms.
        return stdout.rsplit("\0", 1)[0]

    def get_hexsha(self, commitish=None, short=False):
        """Return a hexsha for a given commitish.

        Parameters
        ----------
        commitish : str, optional
          Any identifier that refers to a commit (defaults to "HEAD").
        short : bool, optional
          Return the abbreviated form of the hexsha.

        Returns
        -------
        str or, if there are not commits yet, None.
        """
        stdout = self.format_commit("%{}".format('h' if short else 'H'),
                                    commitish)
        if stdout is not None:
            stdout = stdout.splitlines()
            assert(len(stdout) == 1)
            return stdout[0]

    @normalize_paths(match_return_type=False)
    def get_last_commit_hexsha(self, files):
        """Return the hash of the last commit the modified any of the given
        paths"""
        try:
            commit = self.call_git(
                ['rev-list', '-n1', 'HEAD'],
                files=files,
                expect_fail=True,
            )
            commit = commit.strip()
            return commit if commit else None
        except CommandError:
            if self.get_hexsha() is None:
                # unborn branch, don't freak out
                return None
            raise

    def get_revisions(self, revrange=None, fmt="%H", options=None):
        """Return list of revisions in `revrange`.

        Parameters
        ----------
        revrange : str or list of str or None, optional
            Revisions or revision ranges to walk. If None, revision defaults to
            HEAD unless a revision-modifying option like `--all` or
            `--branches` is included in `options`.
        fmt : string, optional
            Format accepted by `--format` option of `git log`. This should not
            contain new lines because the output is split on new lines.
        options : list of str, optional
            Options to pass to `git log`.  This should not include `--format`.

        Returns
        -------
        List of revisions (str), formatted according to `fmt`.
        """
        if revrange is None:
            revrange = []
        elif isinstance(revrange, str):
            revrange = [revrange]

        cmd = ["git", "log", "--format={}".format(fmt)]
        cmd.extend((options or []) + revrange + ["--"])
        try:
            stdout, _ = self._git_custom_command(None, cmd, expect_fail=True)
        except CommandError as e:
            if "does not have any commits" in e.stderr:
                return []
            raise
        return stdout.splitlines()

    def commit_exists(self, commitish):
        """Does `commitish` exist in the repo?

        Parameters
        ----------
        commitish : str
            A commit or an object that can be dereferenced to one.

        Returns
        -------
        bool
        """
        # Note: The peeling operator "^{commit}" is required so that rev-parse
        # doesn't succeed if passed a full hexsha that is valid but doesn't
        # exist.
        return self.call_git_success(
            ["rev-parse", "--verify", commitish + "^{commit}"])

    def get_merge_base(self, commitishes):
        """Get a merge base hexsha

        Parameters
        ----------
        commitishes: str or list of str
          List of commitishes (branches, hexshas, etc) to determine the merge
          base of. If a single value provided, returns merge_base with the
          current branch.

        Returns
        -------
        str or None
          If no merge-base for given commits, or specified treeish doesn't
          exist, None returned
        """
        if isinstance(commitishes, str):
            commitishes = [commitishes]
        if not commitishes:
            raise ValueError("Provide at least a single value")
        elif len(commitishes) == 1:
            commitishes = commitishes + [self.get_active_branch()]

        try:
            base = self.call_git_oneline(['merge-base'] + commitishes)
        except CommandError as exc:
            if exc.code == 1 and not (exc.stdout or exc.stderr):
                # No merge base was found (unrelated commits).
                return None
            if "fatal: Not a valid object name" in exc.stderr:
                return None
            raise

        return base

    def is_ancestor(self, reva, revb):
        """Is `reva` an ancestor of `revb`?

        Parameters
        ----------
        reva, revb : str
            Revisions.

        Returns
        -------
        bool
        """
        return self.call_git_success(
            ["merge-base", "--is-ancestor", reva, revb])

    def get_commit_date(self, branch=None, date='authored'):
        """Get the date stamp of the last commit (in a branch or head otherwise)

        Parameters
        ----------
        date: {'authored', 'committed'}
          Which date to return.  "authored" will be the date shown by "git show"
          and the one possibly specified via --date to `git commit`

        Returns
        -------
        int or None
          None if no commit
        """
        if date == 'committed':
            format = '%ct'
        elif date == 'authored':
            format = '%at'
        else:
            raise ValueError('unknow date type: {}'.format(date))
        d = self.format_commit(format, commitish=branch)
        return int(d) if d else None

    def get_active_branch(self):
        """Get the name of the active branch

        Returns
        -------
        str or None
          Returns None if there is no active branch, i.e. detached HEAD,
          and the branch name otherwise.
        """
        try:
            out, _ = self._git_custom_command(
                "", ["git", "symbolic-ref", "HEAD"],
                expect_fail=True)
        except CommandError as e:
            if 'HEAD is not a symbolic ref' in e.stderr:
                lgr.debug("detached HEAD in {0}".format(self))
                return None
            else:
                raise e
        return out.strip()[11:]  # strip refs/heads/

    def get_branches(self):
        """Get all branches of the repo.

        Returns
        -------
        [str]
            Names of all branches of this repository.
        """

        return [
            b['refname:strip=2']
            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/heads')
        ]

    def get_remote_branches(self):
        """Get all branches of all remotes of the repo.

        Returns
        -----------
        [str]
            Names of all remote branches.
        """
        # TODO: Reconsider melting with get_branches()

        # TODO: treat entries like this: origin/HEAD -> origin/master'
        # currently this is done in collection

        return [
            b['refname:strip=2']
            for b in self.for_each_ref_(fields='refname:strip=2', pattern='refs/remotes')
        ]

    def get_remotes(self, with_urls_only=False):
        """Get known remotes of the repository

        Parameters
        ----------
        with_urls_only : bool, optional
          return only remotes which have urls

        Returns
        -------
        remotes : list of str
          List of names of the remotes
        """

        # Note: read directly from config and spare instantiation of gitpy.Repo
        # since we need this in AnnexRepo constructor. Furthermore gitpy does it
        # pretty much the same way and the use of a Repo instance seems to have
        # no reason other than a nice object oriented look.
        from datalad.utils import unique

        self.config.reload()
        remotes = unique([x[7:] for x in self.config.sections()
                          if x.startswith("remote.")])

        if with_urls_only:
            remotes = [
                r for r in remotes
                if self.config.get('remote.%s.url' % r)
            ]
        return remotes

    # TODO this is practically unused outside the tests, consider turning
    # into a test helper and trim from the API
    def get_files(self, branch=None):
        """Get a list of files in git.

        Lists the files in the (remote) branch.

        Parameters
        ----------
        branch: str
          Name of the branch to query. Default: active branch.

        Returns
        -------
        [str]
          list of files.
        """
        return [
            str(p.relative_to(self.pathobj))
            for p in self.get_content_info(
                paths=None, ref=branch, untracked='no', eval_file_type=False)
            ]

    @normalize_paths(match_return_type=False)
    def _git_custom_command(self, files, cmd_str,
                            log_stdout=True, log_stderr=True, log_online=False,
                            expect_stderr=True, cwd=None, env=None,
                            shell=None, expect_fail=False,
                            check_fake_dates=False,
                            index_file=None,
                            updates_tree=False):
        """Allows for calling arbitrary commands.

        Helper for developing purposes, i.e. to quickly implement git commands
        for proof of concept without the need to figure out, how this is done
        via GitPython.

        Parameters
        ----------
        files: list of files
        cmd_str: str or list
          arbitrary command str. `files` is appended to that string.
        updates_tree: bool
          whether or not command updates the working tree. If True, triggers
          necessary reevaluations like self.config.reload()

        Returns
        -------
        stdout, stderr
        """

        # ensure cmd_str becomes a well-formed list:
        if isinstance(cmd_str, str):
            cmd = split_cmdline(cmd_str)
        else:
            cmd = cmd_str[:]  # we will modify in-place

        assert(cmd[0] == 'git')
        cmd = cmd[:1] + self._GIT_COMMON_OPTIONS + cmd[1:]

        if check_fake_dates and self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        if index_file:
            env = (env if env is not None else os.environ).copy()
            env['GIT_INDEX_FILE'] = index_file

        # TODO?: wouldn't splitting interfer with above GIT_INDEX_FILE
        #  handling????
        try:
            out, err = run_gitcommand_on_file_list_chunks(
                self.cmd_call_wrapper.run,
                cmd,
                files,
                log_stderr=log_stderr,
                log_stdout=log_stdout,
                log_online=log_online,
                expect_stderr=expect_stderr,
                cwd=cwd,
                env=env,
                shell=shell,
                expect_fail=expect_fail)
        except CommandError as e:
            ignored = re.search(GitIgnoreError.pattern, e.stderr)
            if ignored:
                raise GitIgnoreError(cmd=e.cmd, msg=e.stderr,
                                     code=e.code, stdout=e.stdout,
                                     stderr=e.stderr,
                                     paths=ignored.groups()[0].splitlines())
            raise

        if updates_tree:
            lgr.debug("Reloading config due to supposed working tree update")
            self.config.reload()

        return out, err

    # Convenience wrappers for one-off git calls that don't require further
    # processing or error handling.

    def call_git(self, args, files=None,
                 expect_stderr=False, expect_fail=False):
        """Call git and return standard output.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        expect_fail : bool, optional
          A non-zero exit is expected and should not be elevated above the
          DEBUG level.

        Returns
        -------
        standard output (str)

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        """
        out, _ = self._git_custom_command(files, ["git"] + args,
                                          expect_stderr=expect_stderr,
                                          expect_fail=expect_fail,
                                          check_fake_dates=True)
        return out

    def call_git_items_(self, args, files=None, expect_stderr=False, sep=None):
        """Call git, splitting output on `sep`.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        sep : str, optional
          Split the output by `str.split(sep)` rather than `str.splitlines`.

        Returns
        -------
        Generator that yields output items.

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        """
        out, _ = self._git_custom_command(files, ["git"] + args,
                                          expect_stderr=expect_stderr,
                                          check_fake_dates=True)
        yield from (out.split(sep) if sep else out.splitlines())

    def call_git_oneline(self, args, files=None, expect_stderr=False):
        """Call git for a single line of output.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.
        sep : str, optional
          Split the output by `str.split(sep)` rather than `str.splitlines`.

        Raises
        ------
        CommandError if the call exits with a non-zero status.
        AssertionError if there is more than one line of output.
        """
        lines = list(self.call_git_items_(args, files=files,
                                          expect_stderr=expect_stderr))
        if len(lines) > 1:
            raise AssertionError(
                "Expected {} to return single line, but it returned {}"
                .format(["git"] + args, lines))
        return lines[0]

    def call_git_success(self, args, files=None, expect_stderr=False):
        """Call git and return true if the call exit code of 0.

        Parameters
        ----------
        args : list of str
          Arguments to pass to `git`.
        files : list of str, optional
          File arguments to pass to `git`. The advantage of passing these here
          rather than as part of `args` is that the call will be split into
          multiple calls to avoid exceeding the maximum command line length.
        expect_stderr : bool, optional
          Standard error is expected and should not be elevated above the DEBUG
          level.

        Returns
        -------
        bool
        """
        try:
            self._git_custom_command(files, ["git"] + args,
                                     expect_fail=True,
                                     expect_stderr=expect_stderr,
                                     check_fake_dates=True)
        except CommandError:
            return False
        return True

    def add_remote(self, name, url, options=None):
        """Register remote pointing to a url
        """
        cmd = ['git', 'remote', 'add']
        if options:
            cmd += options
        cmd += [name, url]

        result = self._git_custom_command('', cmd)
        self.config.reload()
        return result

    def remove_remote(self, name):
        """Remove existing remote
        """

        # TODO: testing and error handling!
        from .exceptions import RemoteNotAvailableError
        try:
            out, err = self._git_custom_command(
                '', ['git', 'remote', 'remove', name])
        except CommandError as e:
            if 'fatal: No such remote' in e.stderr:
                raise RemoteNotAvailableError(name,
                                              cmd="git remote remove",
                                              msg="No such remote",
                                              stdout=e.stdout,
                                              stderr=e.stderr)
            else:
                raise e

        # TODO: config.reload necessary?
        self.config.reload()
        return

    def update_remote(self, name=None, verbose=False):
        """
        """
        options = ["-v"] if verbose else []
        name = [name] if name else []
        self._git_custom_command(
            '', ['git', 'remote'] + name + ['update'] + options,
            expect_stderr=True
        )

    def fetch(self, remote=None, refspec=None, all_=False, git_options=None,
              **kwargs):
        """Fetches changes from a remote (or all remotes).

        Parameters
        ----------
        remote : str, optional
          name of the remote to fetch from. If no remote is given and
          `all_` is not set, the tracking branch is fetched.
        refspec : str, optional
          refspec to fetch.
        all_ : bool, optional
          fetch all remotes (and all of their branches).
          Fails if `remote` was given.
        git_options : list, optional
          Additional command line options for git-fetch.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-fetch.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))
        return list(
            self.fetch_(
                remote=remote,
                refspec=refspec,
                all_=all_,
                git_options=git_options,
            )
        )

    @guard_BadName
    def fetch_(self, remote=None, refspec=None, all_=False, git_options=None):
        """Like `fetch`, but returns a generator"""
        yield from self._fetch_push_helper(
            base_cmd=['git', 'fetch', '--verbose', '--progress'],
            action='fetch',
            urlvars=('remote.{}.url', 'remote.{}.url'),
            protocol=GitProgress,
            info_cls=FetchInfo,
            info_from=1,
            add_remote=False,
            remote=remote,
            refspec=refspec,
            all_=all_,
            git_options=git_options)

    # XXX Consider removing this method. It is only used in `update()`,
    # where it could be easily replaced with fetch+merge
    def pull(self, remote=None, refspec=None, git_options=None, **kwargs):
        """Pulls changes from a remote.

        Parameters
        ----------
        remote : str, optional
          name of the remote to pull from. If no remote is given,
          the remote tracking branch is used.
        refspec : str, optional
          refspec to fetch.
        git_options : list, optional
          Additional command line options for git-pull.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-pull.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))

        cmd = ['git', 'pull', '--progress'] + git_options

        if remote is None:
            if refspec:
                # conflicts with using tracking branch or fetch all remotes
                # For now: Just fail.
                # TODO: May be check whether it fits to tracking branch
                raise ValueError(
                    "refspec specified without a remote. ({})".format(refspec))
            # No explicit remote to fetch.
            # => get tracking branch:
            tb_remote, refspec = self.get_tracking_branch()
            if tb_remote is not None:
                remote = tb_remote
            else:
                # No remote, no tracking branch
                # => fail
                raise ValueError("Neither a remote is specified to pull "
                                 "from nor a tracking branch is set up.")

        cmd.append(remote)
        if refspec:
            cmd += ensure_list(refspec)

        # best effort to enable SSH connection caching
        url = self.config.get('remote.{}.url'.format(remote), None)
        if url and is_ssh(url):
            ssh_manager.get_connection(url).open()
        WitlessRunner(
            cwd=self.path,
            env=GitRunner.get_git_environ_adjusted()).run(
                cmd,
                protocol=StdOutCaptureWithGitProgress,
        )

    def push(self, remote=None, refspec=None, all_remotes=False,
             all_=False, git_options=None, **kwargs):
        """Push changes to a remote (or all remotes).

        Parameters
        ----------
        remote : str, optional
          name of the remote to push to. If no remote is given and
          `all_` is not set, the tracking branch is pushed.
        refspec : str, optional
          refspec to push.
        all_ : bool, optional
          push to all remotes. Fails if `remote` was given.
        git_options : list, optional
          Additional command line options for git-push.
        kwargs :
          Deprecated. GitPython-style keyword argument for git-push.
          Will be appended to any git_options.
        """
        git_options = ensure_list(git_options)
        if kwargs:
            git_options.extend(to_options(**kwargs))
        if all_remotes:
            # be nice to the elderly
            all_ = True
        return list(
            self.push_(
                remote=remote,
                refspec=refspec,
                all_=all_,
                git_options=git_options,
            )
        )

    def push_(self, remote=None, refspec=None, all_=False, git_options=None):
        """Like `push`, but returns a generator"""
        yield from self._fetch_push_helper(
            base_cmd=['git', 'push', '--progress', '--porcelain'],
            action='push',
            urlvars=('remote.{}.pushurl', 'remote.{}.url'),
            protocol=StdOutCaptureWithGitProgress,
            info_cls=PushInfo,
            info_from=0,
            add_remote=True,
            remote=remote,
            refspec=refspec,
            all_=all_,
            git_options=git_options)

    def _fetch_push_helper(
            self,
            base_cmd,     # arg list
            action,       # label fetch|push
            urlvars,      # variables to query for URLs
            protocol,     # processor for output
            info_cls,     # Push|FetchInfo
            info_from,    # 0=stdout, 1=stderr
            add_remote,   # whether to add a 'remote' field to the info dict
            remote=None, refspec=None, all_=False, git_options=None):

        git_options = ensure_list(git_options)

        cmd = base_cmd + git_options

        if remote is None:
            if refspec:
                # conflicts with using tracking branch or push all remotes
                # For now: Just fail.
                # TODO: May be check whether it fits to tracking branch
                raise ValueError(
                    "refspec specified without a remote. ({})".format(refspec))
            if all_:
                remotes_to_process = self.get_remotes(with_urls_only=True)
            else:
                # No explicit remote to fetch.
                # => get tracking branch:
                tb_remote, refspec = self.get_tracking_branch()
                if tb_remote is not None:
                    remotes_to_process = [tb_remote]
                else:
                    # No remote, no tracking branch
                    # => fail
                    raise ValueError(
                        "Neither a remote is specified to {} "
                        "from nor a tracking branch is set up.".format(action))
        else:
            if all_:
                raise ValueError(
                    "Option 'all_' conflicts with specified remote "
                    "'{}'.".format(remote))
            remotes_to_process = [remote]

        if refspec:
            # prep for appending to cmd
            refspec = ensure_list(refspec)

        # no need for progress report, when there is just one remote
        log_remote_progress = len(remotes_to_process) > 1
        if log_remote_progress:
            pbar_id = '{}remotes-{}'.format(action, id(self))
            log_progress(
                lgr.info,
                pbar_id,
                'Start %sing remotes for %s', action, self,
                total=len(remotes_to_process),
                label=action.capitalize(),
                unit=' Remotes',
            )
        try:
            for remote in remotes_to_process:
                r_cmd = cmd + [remote]
                if refspec:
                    r_cmd += refspec

                if log_remote_progress:
                    log_progress(
                        lgr.info,
                        pbar_id,
                        '{}ing remote %s'.format(action.capitalize()),
                        remote,
                        update=1,
                        increment=True,
                    )
                # best effort to enable SSH connection caching
                url = self.config.get(
                    # make two attempts to get a URL
                    urlvars[0].format(remote),
                    self.config.get(
                        urlvars[1].format(remote),
                        None)
                )
                if url and is_ssh(url):
                    ssh_manager.get_connection(url).open()
                try:
                    out = WitlessRunner(
                        cwd=self.path,
                        env=GitRunner.get_git_environ_adjusted()).run(
                            r_cmd,
                            protocol=protocol,
                    )
                    output = out[info_from] or ''
                except CommandError as e:
                    # intercept some errors that we express as an error report
                    # in the info dicts
                    if re.match('error: failed to (push|fetch) some refs', e.stderr):
                        output = {1: e.stderr, 0: e.stdout}[info_from]
                        if output is None:
                            output = ''
                    else:
                        raise

                for line in output.splitlines():
                    try:
                        # push info doesn't identify a remote, add it here
                        pi = info_cls._from_line(line)
                        if add_remote:
                            pi['remote'] = remote
                        yield pi
                    except Exception:
                        # it is not progress and no push info
                        # don't hide it completely
                        lgr.debug('git-%s reported: %s', action, line)
        finally:
            if log_remote_progress:
                log_progress(
                    lgr.info,
                    pbar_id,
                    'Finished %sing remotes for %s', action, self,
                )

    def get_remote_url(self, name, push=False):
        """Get the url of a remote.

        Reads the configuration of remote `name` and returns its url or None,
        if there is no url configured.

        Parameters
        ----------
        name: str
          name of the remote
        push: bool
          if True, get the pushurl instead of the fetch url.
        """

        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')
        return self.config.get(var, None)

    def set_remote_url(self, name, url, push=False):
        """Set the URL a remote is pointing to

        Sets the URL of the remote `name`. Requires the remote to already exist.

        Parameters
        ----------
        name: str
          name of the remote
        url: str
        push: bool
          if True, set the push URL, otherwise the fetch URL
        """

        var = 'remote.{0}.{1}'.format(name, 'pushurl' if push else 'url')
        self.config.set(var, url, where='local', reload=True)

    def get_branch_commits_(self, branch=None, limit=None, stop=None):
        """Return commit hexshas for a branch

        Parameters
        ----------
        branch: str, optional
          If not provided, assumes current branch
        limit: None | 'left-only', optional
          Limit which commits to report.  If None -- all commits (merged or not),
          if 'left-only' -- only the commits from the left side of the tree upon
          merges
        stop: str, optional
          hexsha of the commit at which stop reporting (matched one is not
          reported either)

        Yields
        ------
        str
        """
        cmd = ['rev-list']
        if limit == 'left-only':
            cmd.append('--left-only')
        if not branch:
            branch = self.get_active_branch()
        cmd.append(branch)
        for r in self.call_git_items_(cmd):
            if stop and stop == r:
                return
            yield r

    def checkout(self, name, options=None):
        """
        """
        # TODO: May be check for the need of -b options herein?
        cmd = ['git', 'checkout']
        if options:
            cmd += options
        cmd += [str(name)]

        self._git_custom_command('', cmd, expect_stderr=True, updates_tree=True)

    # TODO: Before implementing annex merge, find usages and check for a needed
    # change to call super().merge
    def merge(self, name, options=None, msg=None, allow_unrelated=False, **kwargs):
        if options is None:
            options = []
        if msg:
            options = options + ["-m", msg]
        if allow_unrelated and external_versions['cmd:git'] >= '2.9':
            options += ['--allow-unrelated-histories']
        self._git_custom_command(
            '', ['git', 'merge'] + options + [name],
            check_fake_dates=True,
            **kwargs
        )

    def remove_branch(self, branch):
        self._git_custom_command(
            '', ['git', 'branch', '-D', branch]
        )

    def cherry_pick(self, commit):
        """Cherry pick `commit` to the current branch.

        Parameters
        ----------
        commit : str
            A single commit.
        """
        self._git_custom_command("", ["git", "cherry-pick", commit],
                                 check_fake_dates=True)

    @property
    def dirty(self):
        """Is the repository dirty?

        Note: This provides a quick answer when you simply want to know if
        there are any untracked changes or modifications in this repository or
        its submodules. For finer-grained control and more detailed reporting,
        use status() instead.
        """
        stdout, _ = self._git_custom_command(
            [],
            ["git", "status", "--porcelain",
             # Ensure the result isn't influenced by status.showUntrackedFiles.
             "--untracked-files=normal",
             # Ensure the result isn't influenced by diff.ignoreSubmodules.
             "--ignore-submodules=none"])
        return bool(stdout.strip())

    @property
    def untracked_files(self):
        """Legacy interface, do not use! Use the status() method instead.

        Despite its name, it also reports on untracked datasets, and
        yields their names with trailing path separators.
        """
        return [
            '{}{}'.format(
                str(p.relative_to(self.pathobj)),
                os.sep if props['type'] != 'file' else ''
            )
            for p, props in self.status(
                    untracked='all', eval_submodule_state='no').items()
            if props.get('state', None) == 'untracked'
        ]

    def gc(self, allow_background=False, auto=False):
        """Perform house keeping (garbage collection, repacking)"""
        cmd_options = ['git']
        if not allow_background:
            cmd_options += ['-c', 'gc.autodetach=0']
        cmd_options += ['gc', '--aggressive']
        if auto:
            cmd_options += ['--auto']
        self._git_custom_command('', cmd_options)

    def _parse_gitmodules(self):
        # TODO read .gitconfig from Git blob?
        gitmodules = self.pathobj / '.gitmodules'
        if not gitmodules.exists():
            return {}
        # pull out file content
        out, err = self._git_custom_command(
            '',
            ['git', 'config', '-z', '-l', '--file', '.gitmodules'])
        # abuse our config parser
        db, _ = _parse_gitconfig_dump(out, {}, None, True, cwd=self.path)
        mods = {}
        for k, v in db.items():
            if not k.startswith('submodule.'):
                # we don't know what this is
                lgr.warning("Skip unrecognized .gitmodule specification: %s=%s", k, v)
                continue
            k_l = k.split('.')
            # module name is everything after 'submodule.' that is not the variable
            # name
            mod_name = '.'.join(k_l[1:-1])
            mod = mods.get(mod_name, {})
            # variable name is the last 'dot-free' segment in the key
            mod[k_l[-1]] = v
            mods[mod_name] = mod

        out = {}
        # bring into traditional shape
        for name, props in mods.items():
            if 'path' not in props:
                lgr.warning("Failed to get '%s.path', skipping this submodule", name)
                continue
            modprops = {'gitmodule_{}'.format(k): v
                        for k, v in props.items()
                        if not (k.startswith('__') or k == 'path')}
            modpath = self.pathobj / PurePosixPath(props['path'])
            modprops['gitmodule_name'] = name
            out[modpath] = modprops
        return out

    def get_submodules_(self, paths=None):
        """Yield submodules in this repository.

        Parameters
        ----------
        paths : list(pathlib.PurePath), optional
            Restrict submodules to those under `paths`.

        Returns
        -------
        A generator that yields a dictionary with information for each
        submodule.
        """
        if not (self.pathobj / ".gitmodules").exists():
            return

        modinfo = self._parse_gitmodules()
        for path, props in self.get_content_info(
                paths=paths,
                ref=None,
                untracked='no',
                eval_file_type=False).items():
            if props.get('type', None) != 'dataset':
                continue
            props["path"] = path
            props.update(modinfo.get(path, {}))
            yield props

    def get_submodules(self, sorted_=True, paths=None, compat=True):
        """Return list of submodules.

        Parameters
        ----------
        sorted_ : bool, optional
            Sort submodules by path name.
        paths : list(pathlib.PurePath), optional
            Restrict submodules to those under `paths`.
        compat : bool, optional
            If true, return a namedtuple that incompletely mimics the
            attributes of GitPython's Submodule object in hope of backwards
            compatibility with previous callers. Note that this form should be
            considered temporary and callers should be updated; this flag will
            be removed in a future release.

        Returns
        -------
        List of submodule namedtuples if `compat` is true or otherwise a list
        of dictionaries as returned by `get_submodules_`.
        """
        xs = self.get_submodules_(paths=paths)
        if compat:
            warnings.warn("The attribute-based return value of get_submodules() "
                          "exists for compatibility purposes and will be removed "
                          "in an upcoming release",
                          DeprecationWarning)
            xs = (Submodule(name=p["gitmodule_name"],
                            path=str(p["path"].relative_to(self.pathobj)),
                            url=p["gitmodule_url"])
                  for p in xs)

        if sorted_:
            if compat:
                def key(x):
                    return x.path
            else:
                def key(x):
                    return x["path"]
            xs = sorted(xs, key=key)
        return list(xs)

    def add_submodule(self, path, name=None, url=None, branch=None):
        """Add a new submodule to the repository.

        This will alter the index as well as the .gitmodules file, but will not
        create a new commit.  If the submodule already exists, no matter if the
        configuration differs from the one provided, the existing submodule
        is considered as already added and no further action is performed.

        Parameters
        ----------
        path : str
          repository-relative path at which the submodule should be located, and
          which will be created as required during the repository initialization.
        name : str or None
          name/identifier for the submodule. If `None`, the `path` will be used
          as name.
        url : str or None
          git-clone compatible URL. If `None`, the repository is assumed to
          exist, and the url of the first remote is taken instead. This is
          useful if you want to make an existing repository a submodule of
          another one.
        branch : str or None
          name of branch to be checked out in the submodule. The given branch
          must exist in the remote repository, and will be checked out locally
          as a tracking branch. If `None`, remote HEAD will be checked out.
        """
        if name is None:
            name = Path(path).as_posix()
        # XXX the following should do it, but GitPython will refuse to add a submodule
        # unless you specify a URL that is configured as one of its remotes, or you
        # specify no URL, but the repo has at least one remote.
        # this is stupid, as for us it is valid to not have any remote, because we can
        # still obtain the submodule from a future publication location, based on the
        # parent
        # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch)
        # going git native instead
        cmd = ['submodule', 'add', '--name', name]
        if branch is not None:
            cmd += ['-b', branch]
        if url is None:
            # repo must already exist locally
            subm = GitRepo(op.join(self.path, path), create=False, init=False)
            # check that it has a commit, and refuse
            # to operate on it otherwise, or we would get a bastard
            # submodule that cripples git operations
            if not subm.get_hexsha():
                raise InvalidGitRepositoryError(
                    'cannot add subdataset {} with no commits'.format(subm))
            # make an attempt to configure a submodule source URL based on the
            # discovered remote configuration
            remote, branch = subm.get_tracking_branch()
            url = subm.get_remote_url(remote) if remote else None

        if url is None:
            # had no luck with a remote URL
            if not isabs(path):
                # need to recode into a relative path "URL" in POSIX
                # style, even on windows
                url = posixpath.join(curdir, posix_relpath(path))
            else:
                url = path
        cmd += [url, Path(path).as_posix()]
        self.call_git(cmd)
        # record dataset ID if possible for comprehesive metadata on
        # dataset components within the dataset itself
        subm_id = GitRepo(op.join(self.path, path)).config.get(
            'datalad.dataset.id', None)
        if subm_id:
            self.call_git(
                ['config', '--file', '.gitmodules', '--replace-all',
                 'submodule.{}.datalad-id'.format(name), subm_id])
        # ensure supported setup
        _fixup_submodule_dotgit_setup(self, path)
        # TODO: return value

    def deinit_submodule(self, path, **kwargs):
        """Deinit a submodule

        Parameters
        ----------
        path: str
            path to the submodule; relative to `self.path`
        kwargs:
            see `__init__`
        """

        self._git_custom_command(path,
                                 ['git', 'submodule', 'deinit'] +
                                 to_options(**kwargs))
        # TODO: return value

    def update_submodule(self, path, mode='checkout', init=False):
        """Update a registered submodule.

        This will make the submodule match what the superproject expects by
        cloning missing submodules and updating the working tree of the
        submodules. The "updating" can be done in several ways depending
        on the value of submodule.<name>.update configuration variable, or
        the `mode` argument.

        Parameters
        ----------
        path : str
          Identifies which submodule to operate on by it's repository-relative
          path.
        mode : {checkout, rebase, merge}
          Update procedure to perform. 'checkout': the commit recorded in the
          superproject will be checked out in the submodule on a detached HEAD;
          'rebase': the current branch of the submodule will be rebased onto
          the commit recorded in the superproject; 'merge': the commit recorded
          in the superproject will be merged into the current branch in the
          submodule.
        init : bool
          If True, initialize all submodules for which "git submodule init" has
          not been called so far before updating.
          Primarily provided for internal purposes and should not be used directly
          since would result in not so annex-friendly .git symlinks/references
          instead of full featured .git/ directories in the submodules
        """
        if GitRepo.is_valid_repo(self.pathobj / path):
            subrepo = GitRepo(self.pathobj / path, create=False)
            subbranch = subrepo.get_active_branch() if subrepo else None
            subbranch_hexsha = subrepo.get_hexsha(subbranch) if subrepo else None
        else:
            subrepo = None
            subbranch = None
            subbranch_hexsha = None

        cmd = ['git', 'submodule', 'update', '--%s' % mode]
        if init:
            cmd.append('--init')
            subgitpath = opj(self.path, path, '.git')
            if not exists(subgitpath):
                # TODO:  wouldn't with --init we get all those symlink'ed .git/?
                # At least let's warn
                lgr.warning(
                    "Do not use update_submodule with init=True to avoid git creating "
                    "symlinked .git/ directories in submodules"
                )
            #  yoh: I thought I saw one recently but thought it was some kind of
            #  an artifact from running submodule update --init manually at
            #  some point, but looking at this code now I worry that it was not
        self._git_custom_command(path, cmd)

        if not init:
            return

        # track branch originally cloned, only if we had a valid repo at the start
        updated_subbranch = subrepo.get_active_branch() if subrepo else None
        if subbranch and not updated_subbranch:
            # got into 'detached' mode
            # trace if current state is a predecessor of the branch_hexsha
            lgr.debug(
                "Detected detached HEAD after updating submodule %s which was "
                "in %s branch before", self.path, subbranch)
            detached_hexsha = subrepo.get_hexsha()
            if subrepo.get_merge_base(
                    [subbranch_hexsha, detached_hexsha]) == detached_hexsha:
                # TODO: config option?
                # in all likely event it is of the same branch since
                # it is an ancestor -- so we could update that original branch
                # to point to the state desired by the submodule, and update
                # HEAD to point to that location
                lgr.info(
                    "Submodule HEAD got detached. Resetting branch %s to point "
                    "to %s. Original location was %s",
                    subbranch, detached_hexsha[:8], subbranch_hexsha[:8]
                )
                branch_ref = 'refs/heads/%s' % subbranch
                subrepo.update_ref(branch_ref, detached_hexsha)
                assert(subrepo.get_hexsha(subbranch) == detached_hexsha)
                subrepo.update_ref('HEAD', branch_ref, symbolic=True)
                assert(subrepo.get_active_branch() == subbranch)
            else:
                lgr.warning(
                    "%s has a detached HEAD since cloned branch %s has another common ancestor with %s",
                    subrepo.path, subbranch, detached_hexsha[:8]
                )
        # TODO: return value

    def update_ref(self, ref, value, symbolic=False):
        """Update the object name stored in a ref "safely".

        Just a shim for `git update-ref` call if not symbolic, and
        `git symbolic-ref` if symbolic

        Parameters
        ----------
        ref : str
          Reference, such as `ref/heads/BRANCHNAME` or HEAD.
        value : str
          Value to update to, e.g. hexsha of a commit when updating for a
          branch ref, or branch ref if updating HEAD
        symbolic : None
          To instruct if ref is symbolic, e.g. should be used in case of
          ref=HEAD
        """
        self._git_custom_command(
            '',
            ['git', 'symbolic-ref' if symbolic else 'update-ref', ref, value]
        )

    def tag(self, tag, message=None, commit=None, options=None):
        """Tag a commit

        Parameters
        ----------
        tag : str
          Custom tag label. Must be a valid tag name.
        message : str, optional
          If provided, adds ['-m', <message>] to the list of `git tag`
          arguments.
        commit : str, optional
          If provided, will be appended as last argument to the `git tag` call,
          and can be used to identify the commit that shall be tagged, if
          not HEAD.
        options : list, optional
          Additional command options, inserted prior a potential `commit`
          argument.
        """
        # TODO: call in save.py complains about extensive logging. When does it
        # happen in what way? Figure out, whether to just silence it or raise or
        # whatever else.
        args = ['tag']
        if message:
            args += ['-m', message]
        if options is not None:
            args.extend(options)
        args.append(tag)
        if commit:
            args.append(commit)
        self.call_git(args)

    def get_tags(self, output=None):
        """Get list of tags

        Parameters
        ----------
        output : str, optional
          If given, limit the return value to a list of values matching that
          particular key of the tag properties.

        Returns
        -------
        list
          Each item is a dictionary with information on a tag. At present
          this includes 'hexsha', and 'name', where the latter is the string
          label of the tag, and the former the hexsha of the object the tag
          is attached to. The list is sorted by the creator date (committer
          date for lightweight tags and tagger date for annotated tags), with
          the most recent commit being the last element.
        """
        tags = [
            dict(
                name=t['refname:strip=2'],
                hexsha=t['object'] if t['object'] else t['objectname'],
            )
            for t in self.for_each_ref_(
                fields=['refname:strip=2', 'objectname', 'object'],
                pattern='refs/tags',
                sort='creatordate')
        ]
        if output:
            return [t[output] for t in tags]
        else:
            return tags

    def describe(self, commitish=None, **kwargs):
        """ Quick and dirty implementation to call git-describe

        Parameters
        ----------
        kwargs:
            transformed to cmdline options for git-describe;
            see __init__ for description of the transformation
        """
        # TODO: be more precise what failure to expect when and raise actual
        # errors
        cmd = ['git', 'describe'] + to_options(**kwargs)
        if commitish is not None:
            cmd.append(commitish)
        try:
            describe, outerr = self._git_custom_command(
                [],
                cmd,
                expect_fail=True)
            return describe.strip()
        # TODO: WTF "catch everything"?
        except:
            return None

    def get_tracking_branch(self, branch=None, remote_only=False):
        """Get the tracking branch for `branch` if there is any.

        Parameters
        ----------
        branch: str
            local branch to look up. If none is given, active branch is used.
        remote_only : bool
            Don't return a value if the upstream remote is set to "." (meaning
            this repository).

        Returns
        -------
        tuple
            (remote or None, refspec or None) of the tracking branch
        """
        if branch is None:
            branch = self.get_active_branch()
            if branch is None:
                return None, None

        track_remote = self.config.get('branch.{0}.remote'.format(branch), None)
        if remote_only and track_remote == ".":
            return None, None
        track_branch = self.config.get('branch.{0}.merge'.format(branch), None)
        return track_remote, track_branch

    @property
    def count_objects(self):
        """return dictionary with count, size(in KiB) information of git objects
        """

        count_cmd = ['git', 'count-objects', '-v']
        count_str, err = self._git_custom_command('', count_cmd)
        count = {key: int(value)
                 for key, value in [item.split(': ')
                                    for item in count_str.split('\n')
                                    if len(item.split(': ')) == 2]}
        return count

    def get_git_attributes(self):
        """Query gitattributes which apply to top level directory

        It is a thin compatibility/shortcut wrapper around more versatile
        get_gitattributes which operates on a list of paths and returns
        a dictionary per each path

        Returns
        -------
        dict:
          a dictionary with attribute name and value items relevant for the
          top ('.') directory of the repository, and thus most likely the
          default ones (if not overwritten with more rules) for all files within
          repo.
        """
        return self.get_gitattributes('.')['.']

    def get_gitattributes(self, path, index_only=False):
        """Query gitattributes for one or more paths

        Parameters
        ----------
        path: path or list
          Path(s) to query. Paths may be relative or absolute.
        index_only: bool
          Flag whether to consider only gitattribute setting that are reflected
          in the repository index, not just in the work tree content.

        Returns
        -------
        dict:
          Each key is a queried path (always relative to the repostiory root),
          each value is a dictionary with attribute
          name and value items. Attribute values are either True or False,
          for set and unset attributes, or are the literal attribute value.
        """
        path = ensure_list(path)
        cmd = ["git", "check-attr", "-z", "--all"]
        if index_only:
            cmd.append('--cached')
        stdout, stderr = self._git_custom_command(path, cmd)
        # make sure we have one entry for each query path to
        # simplify work with the result
        attributes = {_normalize_path(self.path, p): {} for p in path}
        attr = []
        for item in stdout.split('\0'):
            attr.append(item)
            if len(attr) < 3:
                continue
            # we have a full record
            p, name, value = attr
            attrs = attributes[p]
            attrs[name] = \
                True if value == 'set' else False if value == 'unset' else value
            # done, reset item
            attr = []
        return attributes

    def set_gitattributes(self, attrs, attrfile='.gitattributes', mode='a'):
        """Set gitattributes

        By default appends additional lines to `attrfile`. Note, that later
        lines in `attrfile` overrule earlier ones, which may or may not be
        what you want. Set `mode` to 'w' to replace the entire file by
        what you provided in `attrs`.

        Parameters
        ----------
        attrs : list
          Each item is a 2-tuple, where the first element is a path pattern,
          and the second element is a dictionary with attribute key/value
          pairs. The attribute dictionary must use the same semantics as those
          returned by `get_gitattributes()`. Path patterns can use absolute paths,
          in which case they will be normalized relative to the directory
          that contains the target .gitattributes file (see `attrfile`).
        attrfile: path
          Path relative to the repository root of the .gitattributes file the
          attributes shall be set in.
        mode: str
          'a' to append .gitattributes, 'w' to replace it
        """

        git_attributes_file = op.join(self.path, attrfile)
        attrdir = op.dirname(git_attributes_file)
        if not op.exists(attrdir):
            os.makedirs(attrdir)
        with open(git_attributes_file, mode) as f:
            for pattern, attr in sorted(attrs, key=lambda x: x[0]):
                # normalize the pattern relative to the target .gitattributes file
                npath = _normalize_path(
                    op.join(self.path, op.dirname(attrfile)), pattern)
                # paths in gitattributes always have to be POSIX
                npath = Path(npath).as_posix()
                attrline = u''
                if npath.count(' '):
                    # quote patterns with spaces
                    attrline += u'"{}"'.format(npath.replace('"', '\\"'))
                else:
                    attrline += npath
                for a in sorted(attr):
                    val = attr[a]
                    if val is True:
                        attrline += ' {}'.format(a)
                    elif val is False:
                        attrline += ' -{}'.format(a)
                    else:
                        attrline += ' {}={}'.format(a, val)
                f.write('\n{}'.format(attrline))

    def get_content_info(self, paths=None, ref=None, untracked='all',
                         eval_file_type=True):
        """Get identifier and type information from repository content.

        This is simplified front-end for `git ls-files/tree`.

        Both commands differ in their behavior when queried about subdataset
        paths. ls-files will not report anything, ls-tree will report on the
        subdataset record. This function uniformly follows the behavior of
        ls-tree (report on the respective subdataset mount).

        Parameters
        ----------
        paths : list(pathlib.PurePath)
          Specific paths, relative to the resolved repository root, to query
          info for. Paths must be normed to match the reporting done by Git,
          i.e. no parent dir components (ala "some/../this").
          If none are given, info is reported for all content.
        ref : gitref or None
          If given, content information is retrieved for this Git reference
          (via ls-tree), otherwise content information is produced for the
          present work tree (via ls-files). With a given reference, the
          reported content properties also contain a 'bytesize' record,
          stating the size of a file in bytes.
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported when no `ref` was given:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_file_type : bool
          If True, inspect file type of untracked files, and report annex
          symlink pointers as type 'file'. This convenience comes with a
          cost; disable to get faster performance if this information
          is not needed.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'

            Note that the reported type will not always match the type of
            content committed to Git, rather it will reflect the nature
            of the content minus platform/mode-specifics. For example,
            a symlink to a locked annexed file on Unix will have a type
            'file', reported, while a symlink to a file in Git or directory
            will be of type 'symlink'.

          `gitshasum`
            SHASUM of the item as tracked by Git, or None, if not
            tracked. This could be different from the SHASUM of the file
            in the worktree, if it was modified.

        Raises
        ------
        ValueError
          In case of an invalid Git reference (e.g. 'HEAD' in an empty
          repository)
        """
        lgr.debug('%s.get_content_info(...)', self)
        # TODO limit by file type to replace code in subdatasets command
        info = OrderedDict()

        if paths:
            # path matching will happen against what Git reports
            # and Git always reports POSIX paths
            # any incoming path has to be relative already, so we can simply
            # convert unconditionally
            paths = [ut.PurePosixPath(p) for p in paths]

        path_strs = list(map(str, paths)) if paths else None

        # this will not work in direct mode, but everything else should be
        # just fine
        if not ref:
            # make sure no operations are pending before we figure things
            # out in the worktree
            self.precommit()

            # --exclude-standard will make sure to honor and standard way
            # git can be instructed to ignore content, and will prevent
            # crap from contaminating untracked file reports
            cmd = ['git', 'ls-files',
                   '--stage', '-z', '-d', '-m', '--exclude-standard']
            # untracked report mode, using labels from `git diff` option style
            if untracked == 'all':
                cmd.append('-o')
            elif untracked == 'normal':
                cmd += ['-o', '--directory', '--no-empty-directory']
            elif untracked == 'no':
                pass
            else:
                raise ValueError(
                    'unknown value for `untracked`: {}'.format(untracked))
            props_re = re.compile(
                r'(?P<type>[0-9]+) (?P<sha>.*) (.*)\t(?P<fname>.*)$')

            if path_strs:
                # we need to get their within repo elements since ls-tree
                # for paths within submodules returns nothing!
                # see https://public-inbox.org/git/20190703193305.GF21553@hopa.kiewit.dartmouth.edu/T/#u
                submodules = [s.path for s in self.get_submodules()]
                path_strs = get_parent_paths(path_strs, submodules)
        else:
            cmd = ['git', 'ls-tree', ref, '-z', '-r', '--full-tree', '-l']
            props_re = re.compile(
                r'(?P<type>[0-9]+) ([a-z]*) (?P<sha>[^ ]*) [\s]*(?P<size>[0-9-]+)\t(?P<fname>.*)$')

        lgr.debug('Query repo: %s', cmd)
        try:
            stdout, stderr = self._git_custom_command(
                path_strs,
                cmd,
                log_stderr=True,
                log_stdout=True,
                # not sure why exactly, but log_online has to be false!
                log_online=False,
                expect_stderr=False,
                shell=False,
                # we don't want it to scream on stdout
                expect_fail=True)
        except CommandError as exc:
            if "fatal: Not a valid object name" in str(exc):
                raise InvalidGitReferenceError(ref)
            raise
        lgr.debug('Done query repo: %s', cmd)

        if not eval_file_type:
            _get_link_target = None
        elif ref:
            def _read_symlink_target_from_catfile(lines):
                # it is always the second line, all checks done upfront
                header = lines.readline()
                if header.rstrip().endswith('missing'):
                    # something we do not know about, should not happen
                    # in real use, but guard against to avoid stalling
                    return ''
                return lines.readline().rstrip()

            _get_link_target = BatchedCommand(
                ['git', 'cat-file', '--batch'],
                path=self.path,
                output_proc=_read_symlink_target_from_catfile,
            )
        else:
            def try_readlink(path):
                try:
                    return os.readlink(path)
                except OSError:
                    # readlink will fail if the symlink reported by ls-files is
                    # not in the working tree (it could be removed or
                    # unlocked). Fall back to a slower method.
                    return op.realpath(path)

            _get_link_target = try_readlink

        try:
            self._get_content_info_line_helper(
                ref,
                info,
                stdout.split('\0'),
                props_re,
                _get_link_target)
        finally:
            if ref and _get_link_target:
                # cancel batch process
                _get_link_target.close()

        lgr.debug('Done %s.get_content_info(...)', self)
        return info

    def _get_content_info_line_helper(self, ref, info, lines,
                                      props_re, get_link_target):
        """Internal helper of get_content_info() to parse Git output"""
        mode_type_map = {
            '100644': 'file',
            '100755': 'file',
            '120000': 'symlink',
            '160000': 'dataset',
        }
        for line in lines:
            if not line:
                continue
            inf = {}
            props = props_re.match(line)
            if not props:
                # not known to Git, but Git always reports POSIX
                path = ut.PurePosixPath(line)
                inf['gitshasum'] = None
            else:
                # again Git reports always in POSIX
                path = ut.PurePosixPath(props.group('fname'))

            # revisit the file props after this path has not been rejected
            if props:
                inf['gitshasum'] = props.group('sha')
                inf['type'] = mode_type_map.get(
                    props.group('type'), props.group('type'))
                if get_link_target and inf['type'] == 'symlink' and \
                        ((ref is None and '.git/annex/objects' in \
                          ut.Path(
                            get_link_target(str(self.pathobj / path))
                          ).as_posix()) or \
                         (ref and \
                          '.git/annex/objects' in get_link_target(
                              u'{}:{}'.format(
                                  ref, str(path))))
                        ):
                    # report annex symlink pointers as file, their
                    # symlink-nature is a technicality that is dependent
                    # on the particular mode annex is in
                    inf['type'] = 'file'

                if ref and inf['type'] == 'file':
                    inf['bytesize'] = int(props.group('size'))

            # join item path with repo path to get a universally useful
            # path representation with auto-conversion and tons of other
            # stuff
            path = self.pathobj.joinpath(path)
            if 'type' not in inf:
                # be nice and assign types for untracked content
                inf['type'] = 'symlink' if path.is_symlink() \
                    else 'directory' if path.is_dir() else 'file'
            info[path] = inf

    def status(self, paths=None, untracked='all', eval_submodule_state='full'):
        """Simplified `git status` equivalent.

        Parameters
        ----------
        paths : list or None
          If given, limits the query to the specified paths. To query all
          paths specify `None`, not an empty list. If a query path points
          into a subdataset, a report is made on the subdataset record
          within the queried dataset only (no recursion).
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_submodule_state : {'full', 'commit', 'no'}
          If 'full' (the default), the state of a submodule is evaluated by
          considering all modifications, with the treatment of untracked files
          determined by `untracked`. If 'commit', the modification check is
          restricted to comparing the submodule's HEAD commit to the one
          recorded in the superdataset. If 'no', the state of the subdataset is
          not evaluated.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'
          `state`
            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.
        """
        lgr.debug('Query status of %r for %s paths',
                  self, len(paths) if paths else 'all')
        return self.diffstatus(
            fr='HEAD' if self.get_hexsha() else None,
            to=None,
            paths=paths,
            untracked=untracked,
            eval_submodule_state=eval_submodule_state)

    def diff(self, fr, to, paths=None, untracked='all',
             eval_submodule_state='full'):
        """Like status(), but reports changes between to arbitrary revisions

        Parameters
        ----------
        fr : str or None
          Revision specification (anything that Git understands). Passing
          `None` considers anything in the target state as new.
        to : str or None
          Revision specification (anything that Git understands), or None
          to compare to the state of the work tree.
        paths : list or None
          If given, limits the query to the specified paths. To query all
          paths specify `None`, not an empty list.
        untracked : {'no', 'normal', 'all'}
          If and how untracked content is reported when `to` is None:
          'no': no untracked files are reported; 'normal': untracked files
          and entire untracked directories are reported as such; 'all': report
          individual files even in fully untracked directories.
        eval_submodule_state : {'full', 'commit', 'no'}
          If 'full' (the default), the state of a submodule is evaluated by
          considering all modifications, with the treatment of untracked files
          determined by `untracked`. If 'commit', the modification check is
          restricted to comparing the submodule's HEAD commit to the one
          recorded in the superdataset. If 'no', the state of the subdataset is
          not evaluated.

        Returns
        -------
        dict
          Each content item has an entry under a pathlib `Path` object instance
          pointing to its absolute path inside the repository (this path is
          guaranteed to be underneath `Repo.path`).
          Each value is a dictionary with properties:

          `type`
            Can be 'file', 'symlink', 'dataset', 'directory'
          `state`
            Can be 'added', 'untracked', 'clean', 'deleted', 'modified'.
        """
        return {k: v for k, v in self.diffstatus(
            fr=fr, to=to, paths=paths,
            untracked=untracked,
            eval_submodule_state=eval_submodule_state).items()
            if v.get('state', None) != 'clean'}

    def diffstatus(self, fr, to, paths=None, untracked='all',
                   eval_submodule_state='full', eval_file_type=True,
                   _cache=None):
        """Like diff(), but reports the status of 'clean' content too"""
        return self._diffstatus(
            fr, to, paths, untracked, eval_submodule_state, eval_file_type,
            _cache)

    def _diffstatus(self, fr, to, paths, untracked, eval_state,
                    eval_file_type, _cache):
        """Just like diffstatus(), but supports an additional evaluation
        state 'global'. If given, it will return a single 'modified'
        (vs. 'clean') state label for the entire repository, as soon as
        it can."""
        def _get_cache_key(label, paths, ref, untracked=None):
            return self.path, label, tuple(paths) if paths else None, \
                ref, untracked

        if _cache is None:
            _cache = {}

        if paths:
            # at this point we must normalize paths to the form that
            # Git would report them, to easy matching later on
            paths = [ut.Path(p) for p in paths]
            paths = [
                p.relative_to(self.pathobj) if p.is_absolute() else p
                for p in paths
            ]

        # TODO report more info from get_content_info() calls in return
        # value, those are cheap and possibly useful to a consumer
        # we need (at most) three calls to git
        if to is None:
            # everything we know about the worktree, including os.stat
            # for each file
            key = _get_cache_key('ci', paths, None, untracked)
            if key in _cache:
                to_state = _cache[key]
            else:
                to_state = self.get_content_info(
                    paths=paths, ref=None, untracked=untracked,
                    eval_file_type=eval_file_type)
                _cache[key] = to_state
            # we want Git to tell us what it considers modified and avoid
            # reimplementing logic ourselves
            key = _get_cache_key('mod', paths, None)
            if key in _cache:
                modified = _cache[key]
            else:
                modified = set(
                    self.pathobj.joinpath(ut.PurePosixPath(p))
                    for p in self._git_custom_command(
                        # low-level code cannot handle pathobjs
                        [str(p) for p in paths] if paths else None,
                        ['git', 'ls-files', '-z', '-m'])[0].split('\0')
                    if p)
                _cache[key] = modified
        else:
            key = _get_cache_key('ci', paths, to)
            if key in _cache:
                to_state = _cache[key]
            else:
                to_state = self.get_content_info(
                    paths=paths, ref=to, eval_file_type=eval_file_type)
                _cache[key] = to_state
            # we do not need worktree modification detection in this case
            modified = None
        # origin state
        key = _get_cache_key('ci', paths, fr)
        if key in _cache:
            from_state = _cache[key]
        else:
            if fr:
                from_state = self.get_content_info(
                    paths=paths, ref=fr, eval_file_type=eval_file_type)
            else:
                # no ref means from nothing
                from_state = {}
            _cache[key] = from_state

        status = OrderedDict()
        for f, to_state_r in to_state.items():
            props = None
            if f not in from_state:
                # this is new, or rather not known to the previous state
                props = dict(
                    state='added' if to_state_r['gitshasum'] else 'untracked',
                )
                if 'type' in to_state_r:
                    props['type'] = to_state_r['type']
            elif to_state_r['gitshasum'] == from_state[f]['gitshasum'] and \
                    (modified is None or f not in modified):
                if to_state_r['type'] != 'dataset':
                    # no change in git record, and no change on disk
                    props = dict(
                        # at this point we know that the reported object ids
                        # for this file are identical in the to and from
                        # records.  If to is None, we're comparing to the
                        # working tree and a deleted file will still have an
                        # identical id, so we need to check whether the file is
                        # gone before declaring it clean. This working tree
                        # check is irrelevant and wrong if to is a ref.
                        state='clean' if to is not None or (f.exists() or \
                              f.is_symlink()) else 'deleted',
                        type=to_state_r['type'],
                    )
                else:
                    # a dataset
                    props = dict(type=to_state_r['type'])
                    if to is not None:
                        # we can only be confident without looking
                        # at the worktree, if we compare to a recorded
                        # state
                        props['state'] = 'clean'
                    else:
                        # report the shasum that we know, for further
                        # wrangling of subdatasets below
                        props['gitshasum'] = to_state_r['gitshasum']
                        props['prev_gitshasum'] = from_state[f]['gitshasum']
            else:
                # change in git record, or on disk
                props = dict(
                    # TODO we could have a new file that is already staged
                    # but had subsequent modifications done to it that are
                    # unstaged. Such file would presently show up as 'added'
                    # ATM I think this is OK, but worth stating...
                    state='modified' if f.exists() or \
                    f.is_symlink() else 'deleted',
                    # TODO record before and after state for diff-like use
                    # cases
                    type=to_state_r['type'],
                )
            state = props.get('state', None)
            if eval_state == 'global' and \
                    state not in ('clean', None):
                # any modification means globally 'modified'
                return 'modified'
            if state in ('clean', 'added', 'modified'):
                props['gitshasum'] = to_state_r['gitshasum']
                if 'bytesize' in to_state_r:
                    # if we got this cheap, report it
                    props['bytesize'] = to_state_r['bytesize']
                elif props['state'] == 'clean' and 'bytesize' in from_state[f]:
                    # no change, we can take this old size info
                    props['bytesize'] = from_state[f]['bytesize']
            if state in ('clean', 'modified', 'deleted'):
                props['prev_gitshasum'] = from_state[f]['gitshasum']
            status[f] = props

        for f, from_state_r in from_state.items():
            if f not in to_state:
                # we new this, but now it is gone and Git is not complaining
                # about it being missing -> properly deleted and deletion
                # stages
                status[f] = dict(
                    state='deleted',
                    type=from_state_r['type'],
                    # report the shasum to distinguish from a plainly vanished
                    # file
                    gitshasum=from_state_r['gitshasum'],
                )
                if eval_state == 'global':
                    return 'modified'

        if to is not None or eval_state == 'no':
            # if we have `to` we are specifically comparing against
            # a recorded state, and this function only attempts
            # to label the state of a subdataset, not investigate
            # specifically what the changes in subdatasets are
            # this is done by a high-level command like rev-diff
            # so the comparison within this repo and the present
            # `state` label are all we need, and they are done already
            if eval_state == 'global':
                return 'clean'
            else:
                return status

        # loop over all subdatasets and look for additional modifications
        for f, st in status.items():
            f = str(f)
            if 'state' in st or not st['type'] == 'dataset':
                # no business here
                continue
            if not GitRepo.is_valid_repo(f):
                # submodule is not present, no chance for a conflict
                st['state'] = 'clean'
                continue
            # we have to recurse into the dataset and get its status
            subrepo = GitRepo(f)
            subrepo_commit = subrepo.get_hexsha()
            st['gitshasum'] = subrepo_commit
            # subdataset records must be labeled clean up to this point
            # test if current commit in subdataset deviates from what is
            # recorded in the dataset
            st['state'] = 'modified' \
                if st['prev_gitshasum'] != subrepo_commit \
                else 'clean'
            if eval_state == 'global' and st['state'] == 'modified':
                return 'modified'
            if eval_state == 'commit':
                continue
            # the recorded commit did not change, so we need to make
            # a more expensive traversal
            st['state'] = subrepo._diffstatus(
                # we can use 'HEAD' because we know that the commit
                # did not change. using 'HEAD' will facilitate
                # caching the result
                fr='HEAD',
                to=None,
                paths=None,
                untracked=untracked,
                eval_state='global',
                eval_file_type=False,
                _cache=_cache) if st['state'] == 'clean' else 'modified'
            if eval_state == 'global' and st['state'] == 'modified':
                return 'modified'

        if eval_state == 'global':
            return 'clean'
        else:
            return status

    def _save_pre(self, paths, _status, **kwargs):
        # helper to get an actionable status report
        if paths is not None and not paths and not _status:
            return
        if _status is None:
            if 'untracked' not in kwargs:
                kwargs['untracked'] = 'normal'
            status = self.status(
                paths=paths,
                **{k: kwargs[k] for k in kwargs
                   if k in ('untracked', 'eval_submodule_state')})
        else:
            # we want to be able to add items down the line
            # make sure to detach from prev. owner
            status = _status.copy()
        status = OrderedDict(
            (k, v) for k, v in status.items()
            if v.get('state', None) != 'clean'
        )
        return status

    def get_staged_paths(self):
        """Returns a list of any stage repository path(s)

        This is a rather fast call, as it will not depend on what is going on
        in the worktree.
        """
        try:
            stdout, stderr = self._git_custom_command(
                None,
                ['git', 'diff', '--name-only', '--staged'],
                cwd=self.path,
                log_stderr=True,
                log_stdout=True,
                log_online=False,
                expect_stderr=False,
                expect_fail=True)
        except CommandError as e:
            lgr.debug(exc_str(e))
            stdout = ''
        return [f for f in stdout.split('\n') if f]

    def _save_post(self, message, status, partial_commit):
        # helper to commit changes reported in status

        # TODO remove pathobj stringification when commit() can
        # handle it
        to_commit = [str(f.relative_to(self.pathobj))
                     for f, props in status.items()] \
                    if partial_commit else None
        if not partial_commit or to_commit:
            # we directly call GitRepo.commit() to avoid a whole slew
            # if direct-mode safeguards and workarounds in the AnnexRepo
            # implementation (which also run an additional dry-run commit
            GitRepo.commit(
                self,
                files=to_commit,
                msg=message,
                options=None,
                # do not raise on empty commit
                # it could be that the `add` in this save-cycle has already
                # brought back a 'modified' file into a clean state
                careless=True,
            )

    def save(self, message=None, paths=None, _status=None, **kwargs):
        """Save dataset content.

        Parameters
        ----------
        message : str or None
          A message to accompany the changeset in the log. If None,
          a default message is used.
        paths : list or None
          Any content with path matching any of the paths given in this
          list will be saved. Matching will be performed against the
          dataset status (GitRepo.status()), or a custom status provided
          via `_status`. If no paths are provided, ALL non-clean paths
          present in the repo status or `_status` will be saved.
        _status : dict or None
          If None, Repo.status() will be queried for the given `ds`. If
          a dict is given, its content will be used as a constraint.
          For example, to save only modified content, but no untracked
          content, set `paths` to None and provide a `_status` that has
          no entries for untracked content.
        **kwargs :
          Additional arguments that are passed to underlying Repo methods.
          Supported:

          - git : bool (passed to Repo.add()
          - eval_submodule_state : {'full', 'commit', 'no'}
            passed to Repo.status()
          - untracked : {'no', 'normal', 'all'} - passed to Repo.satus()
        """
        return list(
            self.save_(
                message=message,
                paths=paths,
                _status=_status,
                **kwargs
            )
        )

    def save_(self, message=None, paths=None, _status=None, **kwargs):
        """Like `save()` but working as a generator."""
        from datalad.interface.results import get_status_dict

        status = self._save_pre(paths, _status, **kwargs)
        if not status:
            # all clean, nothing todo
            lgr.debug('Nothing to save in %r, exiting early', self)
            return

        # three things are to be done:
        # - remove (deleted if not already staged)
        # - add (modified/untracked)
        # - commit (with all paths that have been touched, to bypass
        #   potential pre-staged bits)

        need_partial_commit = True if self.get_staged_paths() else False

        # remove first, because removal of a subds would cause a
        # modification of .gitmodules to be added to the todo list
        to_remove = [
            # TODO remove pathobj stringification when delete() can
            # handle it
            str(f.relative_to(self.pathobj))
            for f, props in status.items()
            if props.get('state', None) == 'deleted' and
            # staged deletions have a gitshasum reported for them
            # those should not be processed as git rm will error
            # due to them being properly gone already
            not props.get('gitshasum', None)]
        vanished_subds = any(
            props.get('type', None) == 'dataset' and
            props.get('state', None) == 'deleted'
            for f, props in status.items())
        if to_remove:
            for r in self.remove(
                    to_remove,
                    # we would always see individual files
                    recursive=False):
                # TODO normalize result
                yield get_status_dict(
                    action='delete',
                    refds=self.pathobj,
                    # TODO make remove() report the type
                    # for now it claims to report on files only
                    type='file',
                    path=(self.pathobj / ut.PurePosixPath(r)),
                    # make remove() report on failures too
                    status='ok',
                    logger=lgr)

        # TODO this additonal query should not be, base on status as given
        # if anyhow possible, however, when paths are given, status may
        # not contain all required information. In case of path=None AND
        # _status=None, we should be able to avoid this, because
        # status should have the full info already
        # looks for contained repositories
        added_submodule = False
        untracked_dirs = [f.relative_to(self.pathobj)
                          for f, props in status.items()
                          if props.get('state', None) == 'untracked' and
                          props.get('type', None) == 'directory']
        to_add_submodules = []
        if untracked_dirs:
            to_add_submodules = [sm for sm, sm_props in
                self.get_content_info(
                    untracked_dirs,
                    ref=None,
                    # request exhaustive list, so that everything that is
                    # still reported as a directory must be its own repository
                    untracked='all').items()
                if sm_props.get('type', None) == 'directory']
            to_add_submodules = _prune_deeper_repos(to_add_submodules)
            for cand_sm in to_add_submodules:
                branch = self.get_active_branch()
                adjusted_match = ADJUSTED_BRANCH_EXPR.match(
                    branch if branch else '')
                try:
                    self.add_submodule(
                        str(cand_sm.relative_to(self.pathobj)),
                        url=None,
                        name=None,
                        branch=adjusted_match.group('name') if adjusted_match
                        else branch
                    )
                except (CommandError, InvalidGitRepositoryError) as e:
                    yield get_status_dict(
                        action='add_submodule',
                        ds=self,
                        path=self.pathobj / ut.PurePosixPath(cand_sm),
                        status='error',
                        message=e.stderr if hasattr(e, 'stderr')
                        else ('not a Git repository: %s', exc_str(e)),
                        logger=lgr)
                    continue
                # This mirrors the result structure yielded for
                # to_stage_submodules below.
                yield get_status_dict(
                    action='add',
                    refds=self.pathobj,
                    type='file',
                    key=None,
                    path=self.pathobj / ut.PurePosixPath(cand_sm),
                    status='ok',
                    logger=lgr)
                added_submodule = True
        if not need_partial_commit:
            # without a partial commit an AnnexRepo would ignore any submodule
            # path in its add helper, hence `git add` them explicitly
            to_stage_submodules = {
                str(f.relative_to(self.pathobj)): props
                for f, props in status.items()
                if props.get('state', None) in ('modified', 'untracked')
                and props.get('type', None) == 'dataset'}
            if to_stage_submodules:
                lgr.debug(
                    '%i submodule path(s) to stage in %r %s',
                    len(to_stage_submodules), self,
                    to_stage_submodules
                    if len(to_stage_submodules) < 10 else '')
                for r in GitRepo._save_add(
                        self,
                        to_stage_submodules,
                        git_opts=None):
                    yield r

        if added_submodule or vanished_subds:
            # need to include .gitmodules in what needs saving
            status[self.pathobj.joinpath('.gitmodules')] = dict(
                type='file', state='modified')
            if hasattr(self, 'annexstatus') and not kwargs.get('git', False):
                # we cannot simply hook into the coming add-call
                # as this would go to annex, so make a dedicted git-add
                # call to ensure .gitmodules is not annexed
                # in any normal DataLad dataset .gitattributes will
                # prevent this, but in a plain repo it won't
                # https://github.com/datalad/datalad/issues/3306
                for r in GitRepo._save_add(
                        self,
                        {op.join(self.path, '.gitmodules'): None}):
                    yield r
        to_add = {
            # TODO remove pathobj stringification when add() can
            # handle it
            str(f.relative_to(self.pathobj)): props
            for f, props in status.items()
            if (props.get('state', None) in ('modified', 'untracked') and
                f not in to_add_submodules)}
        if to_add:
            lgr.debug(
                '%i path(s) to add to %s %s',
                len(to_add), self, to_add if len(to_add) < 10 else '')
            for r in self._save_add(
                    to_add,
                    git_opts=None,
                    **{k: kwargs[k] for k in kwargs
                       if k in (('git',) if hasattr(self, 'annexstatus')
                                else tuple())}):
                yield r

        self._save_post(message, status, need_partial_commit)
        # TODO yield result for commit, prev helper checked hexsha pre
        # and post...

    def _save_add(self, files, git_opts=None):
        """Simple helper to add files in save()"""
        from datalad.interface.results import get_status_dict
        try:
            # without --verbose git 2.9.3  add does not return anything
            add_out = self._git_custom_command(
                list(files.keys()),
                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']
            )
            # get all the entries
            for r in self._process_git_get_output(*add_out):
                yield get_status_dict(
                    action=r.get('command', 'add'),
                    refds=self.pathobj,
                    type='file',
                    path=(self.pathobj / ut.PurePosixPath(r['file']))
                    if 'file' in r else None,
                    status='ok' if r.get('success', None) else 'error',
                    key=r.get('key', None),
                    # while there is no git-annex underneath here, we
                    # tend to fake its behavior, so we can also support
                    # this type of messaging
                    message='\n'.join(r['error-messages'])
                    if 'error-messages' in r else None,
                    logger=lgr)
        except OSError as e:
            lgr.error("add: %s" % e)
            raise


# TODO
# remove submodule: nope, this is just deinit_submodule + remove
# status?


def _fixup_submodule_dotgit_setup(ds, relativepath):
    """Implementation of our current of .git in a subdataset

    Each subdataset/module has its own .git directory where a standalone
    repository would have it. No gitdir files, no symlinks.
    """
    # move .git to superrepo's .git/modules, remove .git, create
    # .git-file
    path = opj(ds.path, relativepath)
    subds_dotgit = opj(path, ".git")

    repo = GitRepo(path, create=False)
    if repo.dot_git.parent == repo.pathobj:
        # this is what we want
        return

    # first we want to remove any conflicting worktree setup
    # done by git to find the checkout at the mountpoint of the
    # submodule, if we keep that, any git command will fail
    # after we move .git
    # Ben: Shouldn't we re-setup a possible worktree afterwards?
    repo.config.unset('core.worktree', where='local')
    # what we have here is some kind of reference, remove and
    # replace by the target
    os.remove(subds_dotgit)
    # make absolute
    src_dotgit = str(repo.dot_git)
    # move .git
    from os import rename, listdir, rmdir
    ensure_dir(subds_dotgit)
    for dot_git_entry in listdir(src_dotgit):
        rename(opj(src_dotgit, dot_git_entry),
               opj(subds_dotgit, dot_git_entry))
    assert not listdir(src_dotgit)
    rmdir(src_dotgit)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user