{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to git-annex by Joey Hess.\n\nFor further information on git-annex see https://git-annex.branchable.com/.\n\n\"\"\"\n\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport shlex\nimport tempfile\nimport time\n\nfrom itertools import chain\nfrom os import linesep\nfrom os.path import curdir\nfrom os.path import join as opj\nfrom os.path import exists\nfrom os.path import islink\nfrom os.path import realpath\nfrom os.path import lexists\nfrom os.path import isdir\nfrom os.path import isabs\nfrom os.path import relpath\nfrom os.path import normpath\nfrom subprocess import Popen, PIPE\nfrom multiprocessing import cpu_count\nfrom weakref import WeakValueDictionary\n\nfrom six import string_types, PY2\nfrom six import iteritems\nfrom six.moves import filter\nfrom git import InvalidGitRepositoryError\n\nfrom datalad import ssh_manager\nfrom datalad.dochelpers import exc_str\nfrom datalad.dochelpers import borrowdoc\nfrom datalad.dochelpers import borrowkwargs\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import nothing_cm\nfrom datalad.utils import auto_repr\nfrom datalad.utils import on_windows\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import assure_list\nfrom datalad.utils import _path_\nfrom datalad.utils import generate_chunks\nfrom datalad.utils import CMD_MAX_ARG\nfrom datalad.utils import assure_unicode, assure_bytes\nfrom datalad.utils import make_tempfile\nfrom datalad.utils import unlink\nfrom datalad.support.json_py import loads as json_loads\nfrom datalad.cmd import GitRunner\n\n# imports from same module:\nfrom .repo import RepoInterface\nfrom .gitrepo import GitRepo\nfrom .gitrepo import NoSuchPathError\nfrom .gitrepo import _normalize_path\nfrom .gitrepo import normalize_path\nfrom .gitrepo import normalize_paths\nfrom .gitrepo import GitCommandError\nfrom .gitrepo import to_options\nfrom . import ansi_colors\nfrom .external_versions import external_versions\nfrom .exceptions import CommandNotAvailableError\nfrom .exceptions import CommandError\nfrom .exceptions import FileNotInAnnexError\nfrom .exceptions import FileInGitError\nfrom .exceptions import FileNotInRepositoryError\nfrom .exceptions import AnnexBatchCommandError\nfrom .exceptions import InsufficientArgumentsError\nfrom .exceptions import OutOfSpaceError\nfrom .exceptions import RemoteNotAvailableError\nfrom .exceptions import OutdatedExternalDependency\nfrom .exceptions import MissingExternalDependency\nfrom .exceptions import IncompleteResultsError\nfrom .exceptions import AccessDeniedError\nfrom .exceptions import AccessFailedError\n\nlgr = logging.getLogger('datalad.annex')\n\n# Limit to # of CPUs and up to 8, but at least 3 to start with\nN_AUTO_JOBS = min(8, max(3, cpu_count()))\n\n\nclass AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget uses\n            # stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)\n\n# TODO: Why was this commented out?\n# @auto_repr\nclass BatchedAnnexes(dict):\n    \"\"\"Class to contain the registry of active batch'ed instances of annex for\n    a repository\n    \"\"\"\n    def __init__(self, batch_size=0, git_options=None):\n        self.batch_size = batch_size\n        self.git_options = git_options or []\n        super(BatchedAnnexes, self).__init__()\n\n    def get(self, codename, annex_cmd=None, **kwargs):\n        if annex_cmd is None:\n            annex_cmd = codename\n\n        git_options = self.git_options + kwargs.pop('git_options', [])\n        if self.batch_size:\n            git_options += ['-c', 'annex.queuesize=%d' % self.batch_size]\n\n        # START RF/BF: extend codename to respect different options the process\n        # is running with\n        # TODO: Eventually there should be more RF'ing, since the actually used\n        # codenames are partially reflecting this already. Any options used\n        # therein should go away, since they are now automatically included.\n        options = kwargs.copy()\n        options['git_options'] = git_options\n        options['annex_cmd'] = annex_cmd\n        for key in options:\n            codename += ':{0}:{1}'.format(key, options[key])\n        # END RF/BF\n\n        if codename not in self:\n            # Create a new git-annex process we will keep around\n            self[codename] = BatchedAnnex(annex_cmd,\n                                          git_options=git_options,\n                                          **kwargs)\n        return self[codename]\n\n    def clear(self):\n        \"\"\"Override just to make sure we don't rely on __del__ to close all\n        the pipes\"\"\"\n        self.close()\n        super(BatchedAnnexes, self).clear()\n\n    def close(self):\n        \"\"\"Close communication to all the batched annexes\n\n        It does not remove them from the dictionary though\n        \"\"\"\n        for p in self.values():\n            p.close()\n\n    def __del__(self):\n        self.close()\n\n\ndef readline_rstripped(stdout):\n    #return iter(stdout.readline, b'').next().rstrip()\n    return stdout.readline().rstrip()\n\n\ndef readlines_until_ok_or_failed(stdout, maxlines=100):\n    \"\"\"Read stdout until line ends with ok or failed\"\"\"\n    out = ''\n    i = 0\n    lgr.log(3, \"Trying to receive from %s\" % stdout)\n    while not stdout.closed:\n        i += 1\n        if maxlines > 0 and i > maxlines:\n            raise IOError(\"Expected no more than %d lines. So far received: %r\" % (maxlines, out))\n        lgr.log(2, \"Expecting a line\")\n        line = stdout.readline()\n        lgr.log(2, \"Received line %r\" % line)\n        out += line\n        if re.match(r'^.*\\b(failed|ok)$', line.rstrip()):\n            break\n    return out.rstrip()\n\n\ndef readline_json(stdout):\n    toload = stdout.readline().strip()\n    return json_loads(toload) if toload else {}\n\n\n@auto_repr\nclass BatchedAnnex(object):\n    \"\"\"Container for an annex process which would allow for persistent communication\n    \"\"\"\n\n    def __init__(self, annex_cmd, git_options=None, annex_options=None, path=None,\n                 json=False,\n                 output_proc=None):\n        if not isinstance(annex_cmd, list):\n            annex_cmd = [annex_cmd]\n        self.annex_cmd = annex_cmd\n        self.git_options = git_options if git_options else []\n        annex_options = annex_options if annex_options else []\n        self.annex_options = annex_options + (['--json'] if json else [])\n        self.path = path\n        if output_proc is None:\n            output_proc = readline_json if json else readline_rstripped\n        self.output_proc = output_proc\n        self._process = None\n        self._stderr_out = None\n        self._stderr_out_fname = None\n\n    def _initialize(self):\n        # TODO -- should get all those options about --debug and --backend which are used/composed\n        # in AnnexRepo class\n        lgr.debug(\"Initiating a new process for %s\" % repr(self))\n        cmd = ['git'] + self.git_options + \\\n              ['annex'] + self.annex_cmd + self.annex_options + ['--batch']  # , '--debug']\n        lgr.log(5, \"Command: %s\" % cmd)\n        # TODO: look into _run_annex_command  to support default options such as --debug\n        #\n        # according to the internet wisdom there is no easy way with subprocess\n        # while avoid deadlocks etc.  We would need to start a thread/subprocess\n        # to timeout etc\n        # kwargs = dict(bufsize=1, universal_newlines=True) if PY3 else {}\n        self._stderr_out, self._stderr_out_fname = tempfile.mkstemp()\n        self._process = Popen(\n            cmd, stdin=PIPE, stdout=PIPE,  stderr=self._stderr_out,\n            env=GitRunner.get_git_environ_adjusted(),\n            cwd=self.path,\n            bufsize=1,\n            universal_newlines=True  # **kwargs\n        )\n\n    def _check_process(self, restart=False):\n        \"\"\"Check if the process was terminated and restart if restart\n\n        Returns\n        -------\n        bool\n          True if process was alive.\n        str\n          stderr if any recorded if was terminated\n        \"\"\"\n        process = self._process\n        ret = True\n        ret_stderr = None\n        if process and process.poll():\n            lgr.warning(\"Process %s was terminated with returncode %s\" % (process, process.returncode))\n            ret_stderr = self.close(return_stderr=True)\n            ret = False\n        if self._process is None and restart:\n            lgr.warning(\"Restarting the process due to previous failure\")\n            self._initialize()\n        return ret, ret_stderr\n\n    def __call__(self, cmds):\n        \"\"\"\n\n        Parameters\n        ----------\n        cmds : str or tuple or list of (str or tuple)\n\n        Returns\n        -------\n        str or list\n          Output received from annex.  list in case if cmds was a list\n        \"\"\"\n        # TODO: add checks -- may be process died off and needs to be reinitiated\n        if not self._process:\n            self._initialize()\n\n        input_multiple = isinstance(cmds, list)\n        if not input_multiple:\n            cmds = [cmds]\n\n        output = []\n\n        for entry in cmds:\n            if not isinstance(entry, string_types):\n                entry = ' '.join(entry)\n            entry = entry + '\\n'\n            lgr.log(5, \"Sending %r to batched annex %s\" % (entry, self))\n            # apparently communicate is just a one time show\n            # stdout, stderr = self._process.communicate(entry)\n            # according to the internet wisdom there is no easy way with subprocess\n            self._check_process(restart=True)\n            process = self._process  # _check_process might have restarted it\n            process.stdin.write(assure_bytes(entry) if PY2 else entry)\n            process.stdin.flush()\n            lgr.log(5, \"Done sending.\")\n            still_alive, stderr = self._check_process(restart=False)\n            # TODO: we might want to handle still_alive, e.g. to allow for\n            #       a number of restarts/resends, but it should be per command\n            #       since for some we cannot just resend the same query. But if\n            #       it is just a \"get\"er - we could resend it few times\n            # We are expecting a single line output\n            # TODO: timeouts etc\n            stdout = assure_unicode(self.output_proc(process.stdout)) \\\n                if not process.stdout.closed else None\n            if stderr:\n                lgr.warning(\"Received output in stderr: %r\", stderr)\n            lgr.log(5, \"Received output: %r\" % stdout)\n            output.append(stdout)\n\n        return output if input_multiple else output[0]\n\n    def __del__(self):\n        self.close()\n\n    def close(self, return_stderr=False):\n        \"\"\"Close communication and wait for process to terminate\n\n        Returns\n        -------\n        str\n          stderr output if return_stderr and stderr file was there.\n          None otherwise\n        \"\"\"\n        ret = None\n        if self._stderr_out:\n            # close possibly still open fd\n            os.fdopen(self._stderr_out).close()\n            self._stderr_out = None\n        if self._process:\n            process = self._process\n            lgr.debug(\n                \"Closing stdin of %s and waiting process to finish\", process)\n            process.stdin.close()\n            process.stdout.close()\n            process.wait()\n            self._process = None\n            lgr.debug(\"Process %s has finished\", process)\n        if self._stderr_out_fname and os.path.exists(self._stderr_out_fname):\n            if return_stderr:\n                with open(self._stderr_out_fname, 'r') as f:\n                    ret = f.read()\n            # remove the file where we kept dumping stderr\n            unlink(self._stderr_out_fname)\n            self._stderr_out_fname = None\n        return ret\n\n\ndef _get_size_from_perc_complete(count, perc):\n    \"\"\"A helper to get full size if know % and corresponding size\"\"\"\n    portion = (float(perc) / 100.)\n    return int(math.ceil(int(count) / portion)) \\\n        if portion else 0\n\n\nclass ProcessAnnexProgressIndicators(object):\n    \"\"\"'Filter' for annex --json output to react to progress indicators\n\n    Instance of this beast should be passed into log_stdout option\n    for git-annex commands runner\n    \"\"\"\n\n    def __init__(self, expected=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        expected: dict, optional\n           key -> size, expected entries (e.g. downloads)\n        \"\"\"\n        # looking forward for multiple downloads at the same time\n        self.pbars = {}\n        self.total_pbar = None\n        self.expected = expected\n        self.only_one_expected = expected and len(self.expected) == 1\n        self._failed = 0\n        self._succeeded = 0\n        self.start()\n\n    def start(self):\n        if self.expected:\n            from datalad.ui import ui\n            total = sum(filter(bool, self.expected.values()))\n            if len(self.expected) > 1:\n                self.total_pbar = ui.get_progressbar(\n                    label=\"Total\", total=total)\n                self.total_pbar.start()\n\n    def __getitem__(self, id_):\n        if self.pbars == 1:\n            # regardless what it is, it is the one!\n            # will happen e.g. in case of add_url_to_file since filename\n            # is not even reported in the status and final msg just\n            # reports the key which is not known before\n            return self.pbars.items()[0]\n        return self.pbars[id_]\n\n    def _update_pbar(self, pbar, new_value):\n        \"\"\"Updates pbar while also updating possibly total pbar\"\"\"\n        old_value = getattr(pbar, '_old_value', 0)\n        # due to http://git-annex.branchable.com/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/?updated\n        # we will just skip the first update to avoid possible incorrect\n        # reporting\n        if not getattr(pbar, '_first_skipped', False):\n            setattr(pbar, '_first_skipped', True)\n            lgr.log(1, \"Skipped first update of pbar %s\", pbar)\n            return\n        setattr(pbar, '_old_value', new_value)\n        diff = new_value - old_value\n        if diff < 0:\n            # so above didn't help!\n            # use warnings not lgr.warn since we apparently swallow stuff\n            # upstairs!  Also it would take care about issuing it only once\n            lgr.debug(\n                \"Got negative diff for progressbar. old_value=%r, new_value=%r\",\n                old_value, new_value)\n            return\n        if self.total_pbar:\n            self.total_pbar.update(diff, increment=True)\n        pbar.update(new_value)\n\n    def _log_info(self, msg):\n        \"\"\"Helper to log a message, so we need to clear up the pbars first\"\"\"\n        if self.total_pbar:\n            self.total_pbar.clear()\n        for pbar in self.pbars.values():\n            pbar.clear()\n        lgr.info(msg)\n\n    def __call__(self, line):\n        try:\n            j = json.loads(line)\n        except:\n            # if we fail to parse, just return this precious thing for\n            # possibly further processing\n            return line\n        # Process some messages which remotes etc might push to us\n        if list(j) == ['info']:\n            # Just INFO was received without anything else -- we log it at INFO\n            info = j['info']\n            if info.startswith('PROGRESS-JSON: '):\n                j_ = json_loads(info[len('PROGRESS-JSON: '):])\n                if ('command' in j_ and 'key' in j_) or 'byte-progress' in j_:\n                    j = j_\n                else:\n                    self._log_info(info)\n            else:\n                self._log_info(info)\n                return\n\n        target_size = None\n        if 'command' in j and ('key' in j or 'file' in j):\n            # might be the finish line message\n            action_item = j.get('key') or j.get('file')\n            j_action_id = (j['command'], action_item)\n            pbar = self.pbars.pop(j_action_id, None)\n            if pbar is None and len(self.pbars) == 1 and self.only_one_expected:\n                # it is the only one left - take it!\n                pbar = self.pbars.popitem()[1]\n            if j.get('success') in {True, 'true'}:\n                self._succeeded += 1\n                if pbar:\n                    self._update_pbar(pbar, pbar.total)\n                elif self.total_pbar:\n                    # we didn't have a pbar for this download, so total should\n                    # get it all at once\n                    try:\n                        target_size = self.expected[action_item]\n                    except:\n                        target_size = None\n                    if not target_size and 'key' in j:\n                        target_size = AnnexRepo.get_size_from_key(j['key'])\n                    self.total_pbar.update(target_size, increment=True)\n            else:\n                lgr.log(5, \"Message with failed status: %s\" % str(j))\n                self._failed += 1\n\n            if self.total_pbar:\n                failed_str = (\n                    \", \" + ansi_colors.color_word(\"%d failed\" % self._failed,\n                                                  ansi_colors.RED)) \\\n                    if self._failed else ''\n\n                self.total_pbar.set_desc(\n                    \"Total (%d ok%s out of %d)\" % (\n                        self._succeeded,\n                        failed_str,\n                        len(self.expected)\n                        if self.expected\n                        else self._succeeded + self._failed))\n                # seems to be of no effect to force it repaint\n                self.total_pbar.refresh()\n\n            if pbar:\n                pbar.finish()\n\n        if 'byte-progress' not in j:\n            # some other thing than progress\n            return line\n\n        # so we have a progress indicator, let's deal with it\n        action = j['action']\n        action_item = action.get('key') or action.get('file')\n        action_id = (action['command'], action_item)\n        if action_id not in self.pbars:\n            # New download!\n            from datalad.ui import ui\n            from datalad.ui import utils as ui_utils\n            # TODO: whenever target size gets reported -- used it!\n            # http://git-annex.branchable.com/todo/interface_to_the___34__progress__34___of_annex_operations/#comment-6bbc26aae9867603863050ddcb09a9a0\n            # for now deduce from key or approx from '%'\n            # TODO: unittest etc to check when we have a relaxed\n            # URL without any size known in advance\n            if not target_size:\n                target_size = \\\n                    AnnexRepo.get_size_from_key(action.get('key')) or \\\n                    _get_size_from_perc_complete(\n                        j['byte-progress'],\n                        j.get('percent-progress', '').rstrip('%')\n                    ) or \\\n                    0\n            w = ui_utils.get_console_width()\n\n            if not action_item and self.only_one_expected:\n                # must be the one!\n                action_item = list(self.expected)[0]\n\n            title = str(action.get('file') or action_item)\n\n            pbar_right = 50\n            title_len = w - pbar_right - 4  # (4 for reserve)\n            if len(title) > title_len:\n                half = title_len//2 - 2\n                title = '%s .. %s' % (title[:half], title[-half:])\n            pbar = self.pbars[action_id] = ui.get_progressbar(\n                label=title, total=target_size)\n            pbar.start()\n\n        lgr.log(1, \"Updating pbar for action_id=%s. annex: %s.\\n\",\n                action_id, j)\n        self._update_pbar(\n            self[action_id],\n            int(j.get('byte-progress'))\n        )\n\n    def finish(self):\n        if self.total_pbar:\n            self.total_pbar.finish()\n            self.total_pbar = None\n        if self.pbars:\n            lgr.warning(\"Still have %d active progress bars when stopping\",\n                        len(self.pbars))\n        for pbar in self.pbars.values():\n            pbar.finish()\n        self.pbars = {}\n        self._failed = 0\n        self._succeeded = 0\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to git-annex by Joey Hess.\n\nFor further information on git-annex see https://git-annex.branchable.com/.\n\n\"\"\"\n\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport shlex\nimport tempfile\nimport time\n\nfrom itertools import chain\nfrom os import linesep\nfrom os.path import curdir\nfrom os.path import join as opj\nfrom os.path import exists\nfrom os.path import islink\nfrom os.path import realpath\nfrom os.path import lexists\nfrom os.path import isdir\nfrom os.path import isabs\nfrom os.path import relpath\nfrom os.path import normpath\nfrom subprocess import Popen, PIPE\nfrom multiprocessing import cpu_count\nfrom weakref import WeakValueDictionary\n\nfrom six import string_types, PY2\nfrom six import iteritems\nfrom six.moves import filter\nfrom git import InvalidGitRepositoryError\n\nfrom datalad import ssh_manager\nfrom datalad.dochelpers import exc_str\nfrom datalad.dochelpers import borrowdoc\nfrom datalad.dochelpers import borrowkwargs\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import nothing_cm\nfrom datalad.utils import auto_repr\nfrom datalad.utils import on_windows\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import assure_list\nfrom datalad.utils import _path_\nfrom datalad.utils import generate_chunks\nfrom datalad.utils import CMD_MAX_ARG\nfrom datalad.utils import assure_unicode, assure_bytes\nfrom datalad.utils import make_tempfile\nfrom datalad.utils import unlink\nfrom datalad.support.json_py import loads as json_loads\nfrom datalad.cmd import GitRunner\n\n# imports from same module:\nfrom .repo import RepoInterface\nfrom .gitrepo import GitRepo\nfrom .gitrepo import NoSuchPathError\nfrom .gitrepo import _normalize_path\nfrom .gitrepo import normalize_path\nfrom .gitrepo import normalize_paths\nfrom .gitrepo import GitCommandError\nfrom .gitrepo import to_options\nfrom . import ansi_colors\nfrom .external_versions import external_versions\nfrom .exceptions import CommandNotAvailableError\nfrom .exceptions import CommandError\nfrom .exceptions import FileNotInAnnexError\nfrom .exceptions import FileInGitError\nfrom .exceptions import FileNotInRepositoryError\nfrom .exceptions import AnnexBatchCommandError\nfrom .exceptions import InsufficientArgumentsError\nfrom .exceptions import OutOfSpaceError\nfrom .exceptions import RemoteNotAvailableError\nfrom .exceptions import OutdatedExternalDependency\nfrom .exceptions import MissingExternalDependency\nfrom .exceptions import IncompleteResultsError\nfrom .exceptions import AccessDeniedError\nfrom .exceptions import AccessFailedError\n\nlgr = logging.getLogger('datalad.annex')\n\n# Limit to # of CPUs and up to 8, but at least 3 to start with\nN_AUTO_JOBS = min(8, max(3, cpu_count()))\n\n\nclass AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget\n            # uses stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)\n\n# TODO: Why was this commented out?\n# @auto_repr\nclass BatchedAnnexes(dict):\n    \"\"\"Class to contain the registry of active batch'ed instances of annex for\n    a repository\n    \"\"\"\n    def __init__(self, batch_size=0, git_options=None):\n        self.batch_size = batch_size\n        self.git_options = git_options or []\n        super(BatchedAnnexes, self).__init__()\n\n    def get(self, codename, annex_cmd=None, **kwargs):\n        if annex_cmd is None:\n            annex_cmd = codename\n\n        git_options = self.git_options + kwargs.pop('git_options', [])\n        if self.batch_size:\n            git_options += ['-c', 'annex.queuesize=%d' % self.batch_size]\n\n        # START RF/BF: extend codename to respect different options the process\n        # is running with\n        # TODO: Eventually there should be more RF'ing, since the actually used\n        # codenames are partially reflecting this already. Any options used\n        # therein should go away, since they are now automatically included.\n        options = kwargs.copy()\n        options['git_options'] = git_options\n        options['annex_cmd'] = annex_cmd\n        for key in options:\n            codename += ':{0}:{1}'.format(key, options[key])\n        # END RF/BF\n\n        if codename not in self:\n            # Create a new git-annex process we will keep around\n            self[codename] = BatchedAnnex(annex_cmd,\n                                          git_options=git_options,\n                                          **kwargs)\n        return self[codename]\n\n    def clear(self):\n        \"\"\"Override just to make sure we don't rely on __del__ to close all\n        the pipes\"\"\"\n        self.close()\n        super(BatchedAnnexes, self).clear()\n\n    def close(self):\n        \"\"\"Close communication to all the batched annexes\n\n        It does not remove them from the dictionary though\n        \"\"\"\n        for p in self.values():\n            p.close()\n\n    def __del__(self):\n        self.close()\n\n\ndef readline_rstripped(stdout):\n    #return iter(stdout.readline, b'').next().rstrip()\n    return stdout.readline().rstrip()\n\n\ndef readlines_until_ok_or_failed(stdout, maxlines=100):\n    \"\"\"Read stdout until line ends with ok or failed\"\"\"\n    out = ''\n    i = 0\n    lgr.log(3, \"Trying to receive from %s\" % stdout)\n    while not stdout.closed:\n        i += 1\n        if maxlines > 0 and i > maxlines:\n            raise IOError(\"Expected no more than %d lines. So far received: %r\" % (maxlines, out))\n        lgr.log(2, \"Expecting a line\")\n        line = stdout.readline()\n        lgr.log(2, \"Received line %r\" % line)\n        out += line\n        if re.match(r'^.*\\b(failed|ok)$', line.rstrip()):\n            break\n    return out.rstrip()\n\n\ndef readline_json(stdout):\n    toload = stdout.readline().strip()\n    return json_loads(toload) if toload else {}\n\n\n@auto_repr\nclass BatchedAnnex(object):\n    \"\"\"Container for an annex process which would allow for persistent communication\n    \"\"\"\n\n    def __init__(self, annex_cmd, git_options=None, annex_options=None, path=None,\n                 json=False,\n                 output_proc=None):\n        if not isinstance(annex_cmd, list):\n            annex_cmd = [annex_cmd]\n        self.annex_cmd = annex_cmd\n        self.git_options = git_options if git_options else []\n        annex_options = annex_options if annex_options else []\n        self.annex_options = annex_options + (['--json'] if json else [])\n        self.path = path\n        if output_proc is None:\n            output_proc = readline_json if json else readline_rstripped\n        self.output_proc = output_proc\n        self._process = None\n        self._stderr_out = None\n        self._stderr_out_fname = None\n\n    def _initialize(self):\n        # TODO -- should get all those options about --debug and --backend which are used/composed\n        # in AnnexRepo class\n        lgr.debug(\"Initiating a new process for %s\" % repr(self))\n        cmd = ['git'] + self.git_options + \\\n              ['annex'] + self.annex_cmd + self.annex_options + ['--batch']  # , '--debug']\n        lgr.log(5, \"Command: %s\" % cmd)\n        # TODO: look into _run_annex_command  to support default options such as --debug\n        #\n        # according to the internet wisdom there is no easy way with subprocess\n        # while avoid deadlocks etc.  We would need to start a thread/subprocess\n        # to timeout etc\n        # kwargs = dict(bufsize=1, universal_newlines=True) if PY3 else {}\n        self._stderr_out, self._stderr_out_fname = tempfile.mkstemp()\n        self._process = Popen(\n            cmd, stdin=PIPE, stdout=PIPE,  stderr=self._stderr_out,\n            env=GitRunner.get_git_environ_adjusted(),\n            cwd=self.path,\n            bufsize=1,\n            universal_newlines=True  # **kwargs\n        )\n\n    def _check_process(self, restart=False):\n        \"\"\"Check if the process was terminated and restart if restart\n\n        Returns\n        -------\n        bool\n          True if process was alive.\n        str\n          stderr if any recorded if was terminated\n        \"\"\"\n        process = self._process\n        ret = True\n        ret_stderr = None\n        if process and process.poll():\n            lgr.warning(\"Process %s was terminated with returncode %s\" % (process, process.returncode))\n            ret_stderr = self.close(return_stderr=True)\n            ret = False\n        if self._process is None and restart:\n            lgr.warning(\"Restarting the process due to previous failure\")\n            self._initialize()\n        return ret, ret_stderr\n\n    def __call__(self, cmds):\n        \"\"\"\n\n        Parameters\n        ----------\n        cmds : str or tuple or list of (str or tuple)\n\n        Returns\n        -------\n        str or list\n          Output received from annex.  list in case if cmds was a list\n        \"\"\"\n        # TODO: add checks -- may be process died off and needs to be reinitiated\n        if not self._process:\n            self._initialize()\n\n        input_multiple = isinstance(cmds, list)\n        if not input_multiple:\n            cmds = [cmds]\n\n        output = []\n\n        for entry in cmds:\n            if not isinstance(entry, string_types):\n                entry = ' '.join(entry)\n            entry = entry + '\\n'\n            lgr.log(5, \"Sending %r to batched annex %s\" % (entry, self))\n            # apparently communicate is just a one time show\n            # stdout, stderr = self._process.communicate(entry)\n            # according to the internet wisdom there is no easy way with subprocess\n            self._check_process(restart=True)\n            process = self._process  # _check_process might have restarted it\n            process.stdin.write(assure_bytes(entry) if PY2 else entry)\n            process.stdin.flush()\n            lgr.log(5, \"Done sending.\")\n            still_alive, stderr = self._check_process(restart=False)\n            # TODO: we might want to handle still_alive, e.g. to allow for\n            #       a number of restarts/resends, but it should be per command\n            #       since for some we cannot just resend the same query. But if\n            #       it is just a \"get\"er - we could resend it few times\n            # We are expecting a single line output\n            # TODO: timeouts etc\n            stdout = assure_unicode(self.output_proc(process.stdout)) \\\n                if not process.stdout.closed else None\n            if stderr:\n                lgr.warning(\"Received output in stderr: %r\", stderr)\n            lgr.log(5, \"Received output: %r\" % stdout)\n            output.append(stdout)\n\n        return output if input_multiple else output[0]\n\n    def __del__(self):\n        self.close()\n\n    def close(self, return_stderr=False):\n        \"\"\"Close communication and wait for process to terminate\n\n        Returns\n        -------\n        str\n          stderr output if return_stderr and stderr file was there.\n          None otherwise\n        \"\"\"\n        ret = None\n        if self._stderr_out:\n            # close possibly still open fd\n            os.fdopen(self._stderr_out).close()\n            self._stderr_out = None\n        if self._process:\n            process = self._process\n            lgr.debug(\n                \"Closing stdin of %s and waiting process to finish\", process)\n            process.stdin.close()\n            process.stdout.close()\n            process.wait()\n            self._process = None\n            lgr.debug(\"Process %s has finished\", process)\n        if self._stderr_out_fname and os.path.exists(self._stderr_out_fname):\n            if return_stderr:\n                with open(self._stderr_out_fname, 'r') as f:\n                    ret = f.read()\n            # remove the file where we kept dumping stderr\n            unlink(self._stderr_out_fname)\n            self._stderr_out_fname = None\n        return ret\n\n\ndef _get_size_from_perc_complete(count, perc):\n    \"\"\"A helper to get full size if know % and corresponding size\"\"\"\n\n    try:\n        portion = (float(perc) / 100.)\n    except ValueError:\n        portion = None\n    return int(math.ceil(int(count) / portion)) \\\n        if portion else 0\n\n\nclass ProcessAnnexProgressIndicators(object):\n    \"\"\"'Filter' for annex --json output to react to progress indicators\n\n    Instance of this beast should be passed into log_stdout option\n    for git-annex commands runner\n    \"\"\"\n\n    def __init__(self, expected=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        expected: dict, optional\n           key -> size, expected entries (e.g. downloads)\n        \"\"\"\n        # looking forward for multiple downloads at the same time\n        self.pbars = {}\n        self.total_pbar = None\n        self.expected = expected\n        self.only_one_expected = expected and len(self.expected) == 1\n        self._failed = 0\n        self._succeeded = 0\n        self.start()\n\n    def start(self):\n        if self.expected:\n            from datalad.ui import ui\n            total = sum(filter(bool, self.expected.values()))\n            if len(self.expected) > 1:\n                self.total_pbar = ui.get_progressbar(\n                    label=\"Total\", total=total)\n                self.total_pbar.start()\n\n    def __getitem__(self, id_):\n        if self.pbars == 1:\n            # regardless what it is, it is the one!\n            # will happen e.g. in case of add_url_to_file since filename\n            # is not even reported in the status and final msg just\n            # reports the key which is not known before\n            return self.pbars.items()[0]\n        return self.pbars[id_]\n\n    def _update_pbar(self, pbar, new_value):\n        \"\"\"Updates pbar while also updating possibly total pbar\"\"\"\n        old_value = getattr(pbar, '_old_value', 0)\n        # due to http://git-annex.branchable.com/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/?updated\n        # we will just skip the first update to avoid possible incorrect\n        # reporting\n        if not getattr(pbar, '_first_skipped', False):\n            setattr(pbar, '_first_skipped', True)\n            lgr.log(1, \"Skipped first update of pbar %s\", pbar)\n            return\n        setattr(pbar, '_old_value', new_value)\n        diff = new_value - old_value\n        if diff < 0:\n            # so above didn't help!\n            # use warnings not lgr.warn since we apparently swallow stuff\n            # upstairs!  Also it would take care about issuing it only once\n            lgr.debug(\n                \"Got negative diff for progressbar. old_value=%r, new_value=%r\",\n                old_value, new_value)\n            return\n        if self.total_pbar:\n            self.total_pbar.update(diff, increment=True)\n        pbar.update(new_value)\n\n    def _log_info(self, msg):\n        \"\"\"Helper to log a message, so we need to clear up the pbars first\"\"\"\n        if self.total_pbar:\n            self.total_pbar.clear()\n        for pbar in self.pbars.values():\n            pbar.clear()\n        lgr.info(msg)\n\n    def __call__(self, line):\n        try:\n            j = json.loads(line)\n        except:\n            # if we fail to parse, just return this precious thing for\n            # possibly further processing\n            return line\n        # Process some messages which remotes etc might push to us\n        if list(j) == ['info']:\n            # Just INFO was received without anything else -- we log it at INFO\n            info = j['info']\n            if info.startswith('PROGRESS-JSON: '):\n                j_ = json_loads(info[len('PROGRESS-JSON: '):])\n                if ('command' in j_ and 'key' in j_) or 'byte-progress' in j_:\n                    j = j_\n                else:\n                    self._log_info(info)\n            else:\n                self._log_info(info)\n                return\n\n        target_size = None\n        if 'command' in j and ('key' in j or 'file' in j):\n            # might be the finish line message\n            action_item = j.get('key') or j.get('file')\n            j_action_id = (j['command'], action_item)\n            pbar = self.pbars.pop(j_action_id, None)\n            if pbar is None and len(self.pbars) == 1 and self.only_one_expected:\n                # it is the only one left - take it!\n                pbar = self.pbars.popitem()[1]\n            if j.get('success') in {True, 'true'}:\n                self._succeeded += 1\n                if pbar:\n                    self._update_pbar(pbar, pbar.total)\n                elif self.total_pbar:\n                    # we didn't have a pbar for this download, so total should\n                    # get it all at once\n                    try:\n                        target_size = self.expected[action_item]\n                    except:\n                        target_size = None\n                    if not target_size and 'key' in j:\n                        target_size = AnnexRepo.get_size_from_key(j['key'])\n                    self.total_pbar.update(target_size, increment=True)\n            else:\n                lgr.log(5, \"Message with failed status: %s\" % str(j))\n                self._failed += 1\n\n            if self.total_pbar:\n                failed_str = (\n                    \", \" + ansi_colors.color_word(\"%d failed\" % self._failed,\n                                                  ansi_colors.RED)) \\\n                    if self._failed else ''\n\n                self.total_pbar.set_desc(\n                    \"Total (%d ok%s out of %d)\" % (\n                        self._succeeded,\n                        failed_str,\n                        len(self.expected)\n                        if self.expected\n                        else self._succeeded + self._failed))\n                # seems to be of no effect to force it repaint\n                self.total_pbar.refresh()\n\n            if pbar:\n                pbar.finish()\n\n        if 'byte-progress' not in j:\n            # some other thing than progress\n            return line\n\n        # so we have a progress indicator, let's deal with it\n        action = j['action']\n        action_item = action.get('key') or action.get('file')\n        action_id = (action['command'], action_item)\n        if action_id not in self.pbars:\n            # New download!\n            from datalad.ui import ui\n            from datalad.ui import utils as ui_utils\n            # TODO: whenever target size gets reported -- used it!\n            # http://git-annex.branchable.com/todo/interface_to_the___34__progress__34___of_annex_operations/#comment-6bbc26aae9867603863050ddcb09a9a0\n            # for now deduce from key or approx from '%'\n            # TODO: unittest etc to check when we have a relaxed\n            # URL without any size known in advance\n            if not target_size:\n                target_size = \\\n                    AnnexRepo.get_size_from_key(action.get('key')) or \\\n                    _get_size_from_perc_complete(\n                        j['byte-progress'],\n                        j.get('percent-progress', '').rstrip('%')\n                    ) or \\\n                    0\n            w = ui_utils.get_console_width()\n\n            if not action_item and self.only_one_expected:\n                # must be the one!\n                action_item = list(self.expected)[0]\n\n            title = str(action.get('file') or action_item)\n\n            pbar_right = 50\n            title_len = w - pbar_right - 4  # (4 for reserve)\n            if len(title) > title_len:\n                half = title_len//2 - 2\n                title = '%s .. %s' % (title[:half], title[-half:])\n            pbar = self.pbars[action_id] = ui.get_progressbar(\n                label=title, total=target_size)\n            pbar.start()\n\n        lgr.log(1, \"Updating pbar for action_id=%s. annex: %s.\\n\",\n                action_id, j)\n        self._update_pbar(\n            self[action_id],\n            int(j.get('byte-progress'))\n        )\n\n    def finish(self):\n        if self.total_pbar:\n            self.total_pbar.finish()\n            self.total_pbar = None\n        if self.pbars:\n            lgr.warning(\"Still have %d active progress bars when stopping\",\n                        len(self.pbars))\n        for pbar in self.pbars.values():\n            pbar.finish()\n        self.pbars = {}\n        self._failed = 0\n        self._succeeded = 0\n",
            "header": {
                "file": {
                    "path": "datalad/support/annexrepo.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "a914b8194",
                "new_commit_hash": "d603e9524",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/annexrepo.py"
            },
            "plus_file": {
                "path": "b/datalad/support/annexrepo.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 2112,
                            "length": 8
                        },
                        "new_range": {
                            "start": 2112,
                            "length": 8
                        },
                        "section": "class AnnexRepo(GitRepo, RepoInterface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                    % str(out_json))",
                                "type": "context"
                            },
                            {
                                "content": "            # Make the output's structure match bcmd's.",
                                "type": "context"
                            },
                            {
                                "content": "            out_json = out_json[0]",
                                "type": "context"
                            },
                            {
                                "content": "            # Don't capture stderr, since download progress provided by wget uses",
                                "type": "deleted"
                            },
                            {
                                "content": "            # stderr.",
                                "type": "deleted"
                            },
                            {
                                "content": "            # Don't capture stderr, since download progress provided by wget",
                                "type": "added"
                            },
                            {
                                "content": "            # uses stderr.",
                                "type": "added"
                            },
                            {
                                "content": "        else:",
                                "type": "context"
                            },
                            {
                                "content": "            options += ['--with-files']",
                                "type": "context"
                            },
                            {
                                "content": "            if backend:",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/annexrepo.py",
                            "type": "class",
                            "name": "AnnexRepo",
                            "content": "class AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget uses\n            # stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 96,
                            "end_lineno": 3423,
                            "parent": null
                        },
                        {
                            "file_name": "datalad/support/annexrepo.py",
                            "type": "method",
                            "name": "AnnexRepo.add_url_to_file",
                            "content": "class AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget uses\n            # stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 2042,
                            "end_lineno": 2145,
                            "parent": {
                                "file_name": "datalad/support/annexrepo.py",
                                "type": "class",
                                "name": "AnnexRepo",
                                "content": "class AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget uses\n            # stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 96,
                                "end_lineno": 3423,
                                "parent": null
                            }
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 3661,
                            "length": 7
                        },
                        "new_range": {
                            "start": 3661,
                            "length": 11
                        },
                        "section": "class BatchedAnnex(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "def _get_size_from_perc_complete(count, perc):",
                                "type": "context"
                            },
                            {
                                "content": "    \"\"\"A helper to get full size if know % and corresponding size\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    portion = (float(perc) / 100.)",
                                "type": "deleted"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "    try:",
                                "type": "added"
                            },
                            {
                                "content": "        portion = (float(perc) / 100.)",
                                "type": "added"
                            },
                            {
                                "content": "    except ValueError:",
                                "type": "added"
                            },
                            {
                                "content": "        portion = None",
                                "type": "added"
                            },
                            {
                                "content": "    return int(math.ceil(int(count) / portion)) \\",
                                "type": "context"
                            },
                            {
                                "content": "        if portion else 0",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/annexrepo.py",
                            "type": "function",
                            "name": "_get_size_from_perc_complete",
                            "content": "def _get_size_from_perc_complete(count, perc):\n    \"\"\"A helper to get full size if know % and corresponding size\"\"\"\n    portion = (float(perc) / 100.)\n    return int(math.ceil(int(count) / portion)) \\\n        if portion else 0",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 3662,
                            "end_lineno": 3666,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class AnnexRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import known_failure_v6\n\nimport logging\nfrom functools import partial\nimport os\nfrom os import mkdir\nfrom os.path import join as opj\nfrom os.path import basename\nfrom os.path import realpath\nfrom os.path import relpath\nfrom os.path import curdir\nfrom os.path import pardir\nfrom os.path import exists\nfrom shutil import copyfile\nfrom nose.tools import assert_not_is_instance\n\nfrom six.moves.urllib.parse import urljoin\nfrom six.moves.urllib.parse import urlsplit\n\nimport git\nfrom git import GitCommandError\nfrom mock import patch\nimport gc\n\nfrom datalad.cmd import Runner\n\nfrom datalad.support.external_versions import external_versions\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.utils import on_windows\nfrom datalad.utils import chpwd\nfrom datalad.utils import rmtree\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import unlink\n\nfrom datalad.tests.utils import ignore_nose_capturing_stdout\nfrom datalad.tests.utils import assert_cwd_unchanged\nfrom datalad.tests.utils import with_testrepos\nfrom datalad.tests.utils import with_tempfile\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import with_batch_direct\nfrom datalad.tests.utils import assert_dict_equal as deq_\nfrom datalad.tests.utils import assert_is_instance\nfrom datalad.tests.utils import assert_false\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_is\nfrom datalad.tests.utils import assert_not_in\nfrom datalad.tests.utils import assert_re_in\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import assert_not_equal\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_true\nfrom datalad.tests.utils import eq_\nfrom datalad.tests.utils import ok_\nfrom datalad.tests.utils import ok_git_config_not_empty\nfrom datalad.tests.utils import ok_annex_get\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_file_has_content\nfrom datalad.tests.utils import swallow_logs\nfrom datalad.tests.utils import swallow_outputs\nfrom datalad.tests.utils import local_testrepo_flavors\nfrom datalad.tests.utils import serve_path_via_http\nfrom datalad.tests.utils import get_most_obscure_supported_name\nfrom datalad.tests.utils import OBSCURE_FILENAME\nfrom datalad.tests.utils import SkipTest\nfrom datalad.tests.utils import skip_ssh\nfrom datalad.tests.utils import find_files\n\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.exceptions import CommandNotAvailableError\nfrom datalad.support.exceptions import FileNotInRepositoryError\nfrom datalad.support.exceptions import FileNotInAnnexError\nfrom datalad.support.exceptions import FileInGitError\nfrom datalad.support.exceptions import OutOfSpaceError\nfrom datalad.support.exceptions import RemoteNotAvailableError\nfrom datalad.support.exceptions import OutdatedExternalDependency\nfrom datalad.support.exceptions import MissingExternalDependency\nfrom datalad.support.exceptions import InsufficientArgumentsError\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.exceptions import IncompleteResultsError\n\nfrom datalad.support.gitrepo import GitRepo\n\n# imports from same module:\nfrom datalad.support.annexrepo import (\n    AnnexRepo,\n    ProcessAnnexProgressIndicators,\n    _get_size_from_perc_complete,\n)\nfrom .utils import check_repo_deals_with_inode_change\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_instance_from_clone(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(dst, '.git', 'annex')))\n\n    # do it again should raise GitCommandError since git will notice\n    # there's already a git-repo at that path and therefore can't clone to `dst`\n    with swallow_logs(new_level=logging.WARN) as cm:\n        assert_raises(GitCommandError, AnnexRepo.clone, src, dst)\n        if git.__version__ != \"1.0.2\" and git.__version__ != \"2.0.5\":\n            assert(\"already exists\" in cm.out)\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_instance_from_existing(path):\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_instance_brand_new(path):\n\n    GitRepo(path)\n    assert_raises(RuntimeError, AnnexRepo, path, create=False)\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_crippled_filesystem(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    # fake git-annex entries in .git/config:\n    writer = ar.repo.config_writer()\n    writer.set_value(\"annex\", \"crippledfilesystem\", True)\n    writer.release()\n    ok_(ar.is_crippled_fs())\n    writer.set_value(\"annex\", \"crippledfilesystem\", False)\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n    # since we can't remove the entry, just rename it to fake its absence:\n    writer.rename_section(\"annex\", \"removed\")\n    writer.set_value(\"annex\", \"something\", \"value\")\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_is_direct_mode(path):\n\n    ar = AnnexRepo(path)\n    eq_(ar.config.getbool(\"annex\", \"direct\", False),\n        ar.is_direct_mode())\n\n\n@with_tempfile()\ndef test_AnnexRepo_is_direct_mode_gitrepo(path):\n    repo = GitRepo(path, create=True)\n    # artificially make .git/annex so no annex section gets initialized\n    # in .git/config.  We did manage somehow to make this happen (via publish)\n    # but didn't reproduce yet, so just creating manually\n    mkdir(opj(repo.path, '.git', 'annex'))\n    ar = AnnexRepo(path, init=False, create=False)\n    # It is unlikely though that annex would be in direct mode (requires explicit)\n    # annex magic, without having annex section under .git/config\n    dm = ar.is_direct_mode()\n\n    if ar.is_crippled_fs() or on_windows:\n        ok_(dm)\n    else:\n        assert_false(dm)\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_set_direct_mode(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available:\n        assert_raises(CommandError, ar.set_direct_mode, True)\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n\n    ar.set_direct_mode(True)\n    ok_(ar.is_direct_mode(), \"Switching to direct mode failed.\")\n    if ar.is_crippled_fs():\n        assert_raises(CommandNotAvailableError, ar.set_direct_mode, False)\n        ok_(\n            ar.is_direct_mode(),\n            \"Indirect mode on crippled fs detected. Shouldn't be possible.\")\n    else:\n        ar.set_direct_mode(False)\n        assert_false(ar.is_direct_mode(), \"Switching to indirect mode failed.\")\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_annex_proxy(src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path)\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available and therefore no 'annex proxy':\n        assert_raises(CommandError, ar.proxy, ['git', 'status'])\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n    ar.set_direct_mode(True)\n\n    # annex proxy raises in indirect mode:\n    try:\n        ar.set_direct_mode(False)\n        assert_raises(CommandNotAvailableError, ar.proxy, ['git', 'status'])\n    except CommandNotAvailableError:\n        # we can't switch to indirect\n        pass\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_key(src, annex_path):\n\n    ar = AnnexRepo.clone(src, annex_path)\n\n    # test-annex.dat should return the correct key:\n    eq_(\n        ar.get_file_key(\"test-annex.dat\"),\n        'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat')\n\n    # and should take a list with an empty string as result, if a file wasn't\n    # in annex:\n    eq_(\n        ar.get_file_key([\"filenotpresent.wtf\", \"test-annex.dat\"]),\n        ['', 'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat']\n    )\n\n    # test.dat is actually in git\n    # should raise Exception; also test for polymorphism\n    assert_raises(IOError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileNotInAnnexError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileInGitError, ar.get_file_key, \"test.dat\")\n\n    # filenotpresent.wtf doesn't even exist\n    assert_raises(IOError, ar.get_file_key, \"filenotpresent.wtf\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_get_outofspace(annex_path):\n    ar = AnnexRepo(annex_path, create=True)\n\n    def raise_cmderror(*args, **kwargs):\n        raise CommandError(\n            cmd=\"whatever\",\n            stderr=\"junk around not enough free space, need 905.6 MB more and after\"\n        )\n\n    with patch.object(AnnexRepo, '_run_annex_command', raise_cmderror) as cma, \\\n            assert_raises(OutOfSpaceError) as cme:\n        ar.get(\"file\")\n    exc = cme.exception\n    eq_(exc.sizemore_msg, '905.6 MB')\n    assert_re_in(\".*annex (find|get). needs 905.6 MB more\", str(exc))\n\n\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_AnnexRepo_get_remote_na(path):\n    ar = AnnexRepo(path)\n\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', options=[\"--from=NotExistingRemote\"])\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n    # and similar one whenever invoking with remote parameter\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', remote=\"NotExistingRemote\")\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n\n# 1 is enough to test file_has_content\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_file_has_content(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n    testfiles = [\"test-annex.dat\", \"test.dat\"]\n\n    eq_(ar.file_has_content(testfiles), [False, False])\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.file_has_content(testfiles, batch=batch), [True, False])\n    eq_(ar.file_has_content(testfiles[:1], batch=batch), [True])\n\n    eq_(ar.file_has_content(testfiles + [\"bogus.txt\"], batch=batch),\n        [True, False, False])\n\n    assert_false(ar.file_has_content(\"bogus.txt\", batch=batch))\n    ok_(ar.file_has_content(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n# 1 is enough to test\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_is_under_annex(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n\n    with open(opj(annex_path, 'not-committed.txt'), 'w') as f:\n        f.write(\"aaa\")\n\n    testfiles = [\"test-annex.dat\", \"not-committed.txt\", \"INFO.txt\"]\n    # wouldn't change\n    target_value = [True, False, False]\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n    eq_(ar.is_under_annex(testfiles[:1], batch=batch), target_value[:1])\n    eq_(ar.is_under_annex(testfiles[1:], batch=batch), target_value[1:])\n\n    eq_(ar.is_under_annex(testfiles + [\"bogus.txt\"], batch=batch),\n                 target_value + [False])\n\n    assert_false(ar.is_under_annex(\"bogus.txt\", batch=batch))\n    ok_(ar.is_under_annex(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_web_remote(sitepath, siteurl, dst):\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = '%s_about.txt' % url_file_prefix\n    testfile2 = '%s_about2.txt' % url_file_prefix\n    testfile3 = opj('d', 'sub.txt')\n\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl])\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # output='full'\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull), set(l))  # the same entries\n    non_web_remote = l[1 - l.index(ar.WEB_UUID)]\n    assert_in('urls', lfull[non_web_remote])\n    eq_(lfull[non_web_remote]['urls'], [])\n    assert_not_in('uuid', lfull[ar.WEB_UUID])  # no uuid in the records\n    eq_(lfull[ar.WEB_UUID]['urls'], [testurl])\n\n    # --all and --key are incompatible\n    assert_raises(CommandError, ar.whereis, [], options='--all', output='full', key=True)\n\n    # output='descriptions'\n    ldesc = ar.whereis(testfile, output='descriptions')\n    eq_(set(ldesc), set([v['description'] for v in lfull.values()]))\n\n    # info w/ and w/o fast mode\n    for fast in [True, False]:\n        info = ar.info(testfile, fast=fast)\n        eq_(info['size'], 14)\n        assert(info['key'])  # that it is there\n        info_batched = ar.info(testfile, batch=True, fast=fast)\n        eq_(info, info_batched)\n        # while at it ;)\n        with swallow_outputs() as cmo:\n            eq_(ar.info('nonexistent', batch=False), None)\n            eq_(ar.info('nonexistent-batch', batch=True), None)\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n            ar.precommit()  # to stop all the batched processes for swallow_outputs\n\n    # annex repo info\n    repo_info = ar.repo_info(fast=False)\n    eq_(repo_info['local annex size'], 14)\n    eq_(repo_info['backend usage'], {'SHA256E': 1})\n    # annex repo info in fast mode\n    repo_info_fast = ar.repo_info(fast=True)\n    # doesn't give much testable info, so just comparing a subset for match with repo_info info\n    eq_(repo_info_fast['semitrusted repositories'], repo_info['semitrusted repositories'])\n    #import pprint; pprint.pprint(repo_info)\n\n    # remove the remote\n    ar.rm_url(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_not_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n\n    # now only 1 copy; drop should fail\n    res = ar.drop(testfile)\n    eq_(res['command'], 'drop')\n    eq_(res['success'], False)\n    assert_in('adjust numcopies', res['note'])\n\n    # read the url using different method\n    ar.add_url_to_file(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # 2 known copies now; drop should succeed\n    ar.drop(testfile)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n    assert_false(ar.file_has_content(testfile))\n    lfull = ar.whereis(testfile, output='full')\n    assert_not_in(non_web_remote, lfull) # not present -- so not even listed\n\n    # multiple files/urls\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl2])\n\n    # TODO: if we ask for whereis on all files, we should get for all files\n    lall = ar.whereis('.')\n    eq_(len(lall), 2)\n    for e in lall:\n        assert(isinstance(e, list))\n    # but we don't know which one for which file. need a 'full' one for that\n    lall_full = ar.whereis('.', output='full')\n    ok_(ar.file_has_content(testfile2))\n    ok_(lall_full[testfile2][non_web_remote]['here'])\n    eq_(set(lall_full), {testfile, testfile2})\n\n    # add a bogus 2nd url to testfile\n\n    someurl = \"http://example.com/someurl\"\n    ar.add_url_to_file(testfile, someurl, options=['--relaxed'])\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull[ar.WEB_UUID]['urls']), {testurl, someurl})\n\n    # and now test with a file in subdirectory\n    subdir = opj(dst, 'd')\n    os.mkdir(subdir)\n    with swallow_outputs() as cmo:\n        ar.add_url_to_file(testfile3, url=testurl3)\n    ok_file_has_content(opj(dst, testfile3), 'more stuff')\n    eq_(set(ar.whereis(testfile3)), {ar.WEB_UUID, non_web_remote})\n    eq_(set(ar.whereis(testfile3, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n\n    # and if we ask for both files\n    info2 = ar.info([testfile, testfile3])\n    eq_(set(info2), {testfile, testfile3})\n    eq_(info2[testfile3]['size'], 10)\n\n    full = ar.whereis([], options='--all', output='full')\n    eq_(len(full.keys()), 3)  # we asked for all files -- got 3 keys\n    assert_in(ar.WEB_UUID, full['SHA256E-s10--a978713ea759207f7a6f9ebc9eaebd1b40a69ae408410ddf544463f6d33a30e1.txt'])\n\n    # which would work even if we cd to that subdir, but then we should use explicit curdir\n    with chpwd(subdir):\n        cur_subfile = opj(curdir, 'sub.txt')\n        eq_(set(ar.whereis(cur_subfile)), {ar.WEB_UUID, non_web_remote})\n        eq_(set(ar.whereis(cur_subfile, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n        testfiles = [cur_subfile, opj(pardir, testfile)]\n        info2_ = ar.info(testfiles)\n        # Should maintain original relative file names\n        eq_(set(info2_), set(testfiles))\n        eq_(info2_[cur_subfile]['size'], 10)\n\n\n@with_tree(tree={\"a.txt\": \"a\",\n                 \"b\": \"b\",\n                 OBSCURE_FILENAME: \"c\",\n                 \"subdir\": {\"d\": \"d\", \"e\": \"e\"}})\ndef test_find_batch_equivalence(path):\n    ar = AnnexRepo(path)\n    files = [\"a.txt\", \"b\", OBSCURE_FILENAME]\n    ar.add(files + [\"subdir\"])\n    ar.commit(\"add files\")\n    query = [\"not-there\"] + files\n    expected = {f: f for f in files}\n    expected.update({\"not-there\": \"\"})\n    eq_(expected, ar.find(query, batch=True))\n    eq_(expected, ar.find(query))\n    # If we give a subdirectory, we split that output.\n    eq_(set(ar.find([\"subdir\"])[\"subdir\"]), {\"subdir/d\", \"subdir/e\"})\n    eq_(ar.find([\"subdir\"]), ar.find([\"subdir\"], batch=True))\n\n\n@with_tempfile(mkdir=True)\ndef test_repo_info(path):\n    repo = AnnexRepo(path)\n    info = repo.repo_info()  # works in empty repo without crashing\n    eq_(info['local annex size'], 0)\n    eq_(info['size of annexed files in working tree'], 0)\n\n    def get_custom(custom={}):\n        \"\"\"Need a helper since repo_info modifies in place so we should generate\n        new each time\n        \"\"\"\n        custom_json = {\n            'available local disk space': 'unknown',\n            'size of annexed files in working tree': \"0\",\n            'success': True,\n            'command': 'info',\n        }\n        if custom:\n            custom_json.update(custom)\n        return [custom_json]\n\n    with patch.object(\n            repo, '_run_annex_command_json',\n            return_value=get_custom()):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], None)\n\n    with patch.object(\n        repo, '_run_annex_command_json',\n        return_value=get_custom({\n            \"available local disk space\": \"19193986496 (+100000 reserved)\"})):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], 19193986496)\n\n\n@with_testrepos('.*annex.*', flavors=['local', 'network'])\n@with_tempfile\ndef test_AnnexRepo_migrating_backends(src, dst):\n    ar = AnnexRepo.clone(src, dst, backend='MD5')\n    eq_(ar.default_backends, ['MD5'])\n    # GitPython has a bug which causes .git/config being wiped out\n    # under Python3, triggered by collecting its config instance I guess\n    gc.collect()\n    ok_git_config_not_empty(ar)  # Must not blow, see https://github.com/gitpython-developers/GitPython/issues/333\n\n    filename = get_most_obscure_supported_name()\n    filename_abs = os.path.join(dst, filename)\n    f = open(filename_abs, 'w')\n    f.write(\"What to write?\")\n    f.close()\n\n    ar.add(filename, backend='MD5')\n    eq_(ar.get_file_backend(filename), 'MD5')\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n\n    # migrating will only do, if file is present\n    ok_annex_get(ar, 'test-annex.dat')\n\n    if ar.is_direct_mode():\n        # No migration in direct mode\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat')\n    else:\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n        ar.migrate_backend('test-annex.dat')\n        eq_(ar.get_file_backend('test-annex.dat'), 'MD5')\n\n        ar.migrate_backend('', backend='SHA1')\n        eq_(ar.get_file_backend(filename), 'SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n\n\ntree1args = dict(\n    tree=(\n        ('firstfile', 'whatever'),\n        ('secondfile', 'something else'),\n        ('remotefile', 'pretends to be remote'),\n        ('faraway', 'incredibly remote')),\n)\n\n# keys for files if above tree is generated and added to annex with MD5E backend\ntree1_md5e_keys = {\n    'firstfile': 'MD5E-s8--008c5926ca861023c1d2a36653fd88e2',\n    'faraway': 'MD5E-s17--5b849ed02f914d3bbb5038fe4e3fead9',\n    'secondfile': 'MD5E-s14--6c7ba9c5a141421e1c03cb9807c97c74',\n    'remotefile': 'MD5E-s21--bf7654b3de20d5926d407ea7d913deb0'\n}\n\n\n@with_tree(**tree1args)\ndef __test_get_md5s(path):\n    # was used just to generate above dict\n    annex = AnnexRepo(path, init=True, backend='MD5E')\n    files = [basename(f) for f in find_files('.*', path)]\n    annex.add(files)\n    annex.commit()\n    print({f: annex.get_file_key(f) for f in files})\n\n\n@with_batch_direct\n@with_tree(**tree1args)\ndef test_dropkey(batch, direct, path):\n    kw = {'batch': batch}\n    annex = AnnexRepo(path, init=True, backend='MD5E', direct=direct)\n    files = list(tree1_md5e_keys)\n    annex.add(files)\n    annex.commit()\n    # drop one key\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # drop multiple\n    annex.drop_key([tree1_md5e_keys[f] for f in files[1:3]], **kw)\n    # drop already dropped -- should work as well atm\n    # https://git-annex.branchable.com/bugs/dropkey_--batch_--json_--force_is_always_succesfull\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # and a mix with already dropped or not\n    annex.drop_key(list(tree1_md5e_keys.values()), **kw)\n\n\n@with_tree(**tree1args)\n@serve_path_via_http()\ndef test_AnnexRepo_backend_option(path, url):\n    ar = AnnexRepo(path, backend='MD5')\n\n    # backend recorded in .gitattributes\n    eq_(ar.get_gitattributes('.')['.']['annex.backend'], 'MD5')\n\n    ar.add('firstfile', backend='SHA1')\n    ar.add('secondfile')\n    eq_(ar.get_file_backend('firstfile'), 'SHA1')\n    eq_(ar.get_file_backend('secondfile'), 'MD5')\n\n    with swallow_outputs() as cmo:\n        # must be added under different name since annex 20160114\n        ar.add_url_to_file('remotefile2', url + 'remotefile', backend='SHA1')\n    eq_(ar.get_file_backend('remotefile2'), 'SHA1')\n\n    with swallow_outputs() as cmo:\n        ar.add_urls([url + 'faraway'], backend='SHA1')\n    # TODO: what's the annex-generated name of this?\n    # For now, workaround:\n    ok_(ar.get_file_backend(f) == 'SHA1'\n        for f in ar.get_indexed_files() if 'faraway' in f)\n\n\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_backend(src, dst):\n    #init local test-annex before cloning:\n    AnnexRepo(src)\n\n    ar = AnnexRepo.clone(src, dst)\n\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n    if not ar.is_direct_mode():\n        # no migration in direct mode\n        ok_annex_get(ar, 'test-annex.dat', network=False)\n        ar.migrate_backend('test-annex.dat', backend='SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n    else:\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat', backend='SHA1')\n\n\n@with_tempfile\ndef test_AnnexRepo_always_commit(path):\n\n    repo = AnnexRepo(path)\n    runner = Runner(cwd=path)\n    file1 = get_most_obscure_supported_name() + \"_1\"\n    file2 = get_most_obscure_supported_name() + \"_2\"\n    with open(opj(path, file1), 'w') as f:\n        f.write(\"First file.\")\n    with open(opj(path, file2), 'w') as f:\n        f.write(\"Second file.\")\n\n    # always_commit == True is expected to be default\n    repo.add(file1)\n\n    # Now git-annex log should show the addition:\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 1)\n    assert_in(file1, out_list[0])\n    # check git log of git-annex branch:\n    # expected: initial creation, update (by annex add) and another\n    # update (by annex log)\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = False\n    repo.add(file2)\n\n    # No additional git commit:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = True\n\n    # Still one commit only in git-annex log,\n    # but 'git annex log' was called when always_commit was true again,\n    # so it should commit the addition at the end. Calling it again should then\n    # show two commits.\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(\"recording state in git\", out_list[1])\n\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(file2, out_list[1])\n\n    # Now git knows as well:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 4)\n\n\n@with_testrepos('basic_annex', flavors=['local'])\n@with_tempfile\ndef test_AnnexRepo_on_uninited_annex(origin, path):\n    # \"Manually\" clone to avoid initialization:\n    from datalad.cmd import Runner\n    runner = Runner()\n    _ = runner([\"git\", \"clone\", origin, path], expect_stderr=True)\n\n    assert_false(exists(opj(path, '.git', 'annex'))) # must not be there for this test to be valid\n    annex = AnnexRepo(path, create=False, init=False)  # so we can initialize without\n    # and still can get our things\n    assert_false(annex.file_has_content('test-annex.dat'))\n    with swallow_outputs():\n        annex.get('test-annex.dat')\n        ok_(annex.file_has_content('test-annex.dat'))\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_commit(path):\n\n    ds = AnnexRepo(path, create=True)\n    filename = opj(path, get_most_obscure_supported_name())\n    with open(filename, 'w') as f:\n        f.write(\"File to add to git\")\n    ds.add(filename, git=True)\n\n    assert_raises(AssertionError, ok_clean_git, path, annex=True)\n\n    ds.commit(\"test _commit\")\n    ok_clean_git(path, annex=True)\n\n    # nothing to commit doesn't raise by default:\n    ds.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, ds.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(opj(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"not-existing\")\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_annex(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    filename_abs = opj(repo.path, filename)\n    with open(filename_abs, \"w\") as f:\n        f.write(\"some\")\n\n    out_json = repo.add(filename)\n    # file is known to annex:\n    if not repo.is_direct_mode():\n        assert_true(os.path.islink(filename_abs),\n                    \"Annexed file is not a link.\")\n    else:\n        assert_false(os.path.islink(filename_abs),\n                     \"Annexed file is link in direct mode.\")\n    assert_in('key', out_json)\n    key = repo.get_file_key(filename)\n    assert_false(key == '')\n    assert_equal(key, out_json['key'])\n    ok_(repo.file_has_content(filename))\n\n    # uncommitted:\n    # but not in direct mode branch\n    if repo.is_direct_mode():\n        ok_(not repo.is_dirty(submodules=False))\n    else:\n        ok_(repo.is_dirty(submodules=False))\n\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename)\n    repo.commit(msg=\"Added another file to annex.\")\n    # known to annex:\n    ok_(repo.get_file_key(filename))\n    ok_(repo.file_has_content(filename))\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_git(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"some\")\n    repo.add(filename, git=True)\n\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n    # uncommitted:\n    ok_(repo.is_dirty(submodules=False))\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename, git=True)\n    repo.commit(msg=\"Added another file to annex.\")\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_AnnexRepo_add_unexpected_direct_mode(path):\n    # tests a special case where a submodule is in direct mode, while it's\n    # superproject is not.\n    # There is no point in this test, if direct mode was enforced in the\n    # superproject already (either by test run configuration or FS) or if the\n    # repositories are in V6 by default (where there is no direct mode)\n\n    top = AnnexRepo(path)\n\n    if top.is_direct_mode() or top.config.get(\"annex.version\") == '6':\n        raise SkipTest(\"Nothing to test for\")\n\n    top.update_submodule('subm 1', init=True)\n    sub = AnnexRepo(opj(path, 'subm 1'))\n    sub.set_direct_mode(True)\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        top.add('.')\n        cml.assert_logged(msg=\"Known bug in direct mode.\",\n                          level=\"WARNING\",\n                          regex=False)\n\n\n\n@ignore_nose_capturing_stdout\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_AnnexRepo_get(src, dst):\n\n    annex = AnnexRepo.clone(src, dst)\n    assert_is_instance(annex, AnnexRepo, \"AnnexRepo was not created.\")\n    testfile = 'test-annex.dat'\n    testfile_abs = opj(dst, testfile)\n    assert_false(annex.file_has_content(\"test-annex.dat\"))\n    with swallow_outputs():\n        annex.get(testfile)\n    ok_(annex.file_has_content(\"test-annex.dat\"))\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n    called = []\n    # for some reason yoh failed mock to properly just call original func\n    orig_run = annex._run_annex_command\n\n    def check_run(cmd, annex_options, **kwargs):\n        called.append(cmd)\n        if cmd == 'find':\n            assert_not_in('-J5', annex_options)\n        elif cmd == 'get':\n            assert_in('-J5', annex_options)\n        else:\n            raise AssertionError(\n                \"no other commands so far should be ran. Got %s, %s\" %\n                (cmd, annex_options)\n            )\n        return orig_run(cmd, annex_options=annex_options, **kwargs)\n\n    annex.drop(testfile)\n    with patch.object(AnnexRepo, '_run_annex_command',\n                      side_effect=check_run, auto_spec=True), \\\n            swallow_outputs():\n        annex.get(testfile, jobs=5)\n    eq_(called, ['find', 'get'])\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n\n# TODO:\n#def init_remote(self, name, options):\n#def enable_remote(self, name):\n\n@with_testrepos('basic_annex$', flavors=['clone'])\n@with_tempfile\ndef _test_AnnexRepo_get_contentlocation(batch, path, work_dir_outside):\n    annex = AnnexRepo(path, create=False, init=False)\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n    # TODO: see if we can avoid this or specify custom exception\n    eq_(annex.get_contentlocation(key, batch=batch), '')\n\n    with swallow_outputs() as cmo:\n        annex.get(fname)\n    key_location = annex.get_contentlocation(key, batch=batch)\n    assert(key_location)\n    # they both should point to the same location eventually\n    eq_(os.path.realpath(opj(annex.path, fname)),\n        os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a subdir of the annex:\n    with chpwd(opj(annex.path, 'subdir'), mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a dir outside of the annex:\n    with chpwd(work_dir_outside, mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n\ndef test_AnnexRepo_get_contentlocation():\n    for batch in (False, True):\n        yield _test_AnnexRepo_get_contentlocation, batch\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('about2_.txt', 'more abouts_'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_to_file_batched(sitepath, siteurl, dst):\n\n    if os.environ.get('DATALAD_FAKE__DATES'):\n        raise SkipTest(\n            \"Faked dates are enabled; skipping batched addurl tests\")\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl2_ = urljoin(siteurl, 'about2_.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = 'about.txt'\n    testfile2 = 'about2.txt'\n    testfile2_ = 'about2_.txt'\n    testfile3 = opj('d', 'sub.txt')\n\n    # add to an existing but not committed file\n    # TODO: __call__ of the BatchedAnnex must be checked to be called\n    copyfile(opj(sitepath, 'about.txt'), opj(dst, testfile))\n    # must crash sensibly since file exists, we shouldn't addurl to non-annexed files\n    with assert_raises(AnnexBatchCommandError):\n        ar.add_url_to_file(testfile, testurl, batch=True)\n\n    # Remove it and re-add\n    unlink(opj(dst, testfile))\n    ar.add_url_to_file(testfile, testurl, batch=True)\n\n    info = ar.info(testfile)\n    eq_(info['size'], 14)\n    assert(info['key'])\n    # not even added to index yet since we this repo is with default batch_size\n    # but: in direct mode it is added!\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n\n    # TODO: none of the below should re-initiate the batch process\n\n    # add to an existing and staged annex file\n    copyfile(opj(sitepath, 'about2.txt'), opj(dst, testfile2))\n    ar.add(testfile2)\n    ar.add_url_to_file(testfile2, testurl2, batch=True)\n    assert(ar.info(testfile2))\n    # not committed yet\n    # assert_in(ar.WEB_UUID, ar.whereis(testfile2))\n\n    # add to an existing and committed annex file\n    copyfile(opj(sitepath, 'about2_.txt'), opj(dst, testfile2_))\n    ar.add(testfile2_)\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n    ar.commit(\"added about2_.txt and there was about2.txt lingering around\")\n    # commit causes closing all batched annexes, so testfile gets committed\n    assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    assert(not ar.dirty)\n    ar.add_url_to_file(testfile2_, testurl2_, batch=True)\n    assert(ar.info(testfile2_))\n    assert_in(ar.WEB_UUID, ar.whereis(testfile2_))\n\n    # add into a new file\n    # filename = 'newfile.dat'\n    filename = get_most_obscure_supported_name()\n\n    # Note: The following line was necessary, since the test setup just\n    # doesn't work with singletons\n    # TODO: Singleton mechanic needs a general solution for this\n    AnnexRepo._unique_instances.clear()\n    ar2 = AnnexRepo(dst, batch_size=1)\n\n    with swallow_outputs():\n        eq_(len(ar2._batched), 0)\n        ar2.add_url_to_file(filename, testurl, batch=True)\n        eq_(len(ar2._batched), 1)  # we added one more with batch_size=1\n        ar2.precommit()  # to possibly stop batch process occupying the stdout\n    ar2.commit(\"added new file\")  # would do nothing ATM, but also doesn't fail\n    assert_in(filename, ar2.get_files())\n    assert_in(ar.WEB_UUID, ar2.whereis(filename))\n\n    if not ar.is_direct_mode():\n        # in direct mode there's nothing to commit\n        ar.commit(\"actually committing new files\")\n    assert_in(filename, ar.get_files())\n    assert_in(ar.WEB_UUID, ar.whereis(filename))\n    # this poor bugger still wasn't added since we used default batch_size=0 on him\n\n    # and closing the pipes now shoudn't anyhow affect things\n    eq_(len(ar._batched), 1)\n    ar._batched.close()\n    eq_(len(ar._batched), 1)  # doesn't remove them, just closes\n    assert(not ar.dirty)\n\n    ar._batched.clear()\n    eq_(len(ar._batched), 0)  # .clear also removes\n\n    raise SkipTest(\"TODO: more, e.g. add with a custom backend\")\n    # TODO: also with different modes (relaxed, fast)\n    # TODO: verify that file is added with that backend and that we got a new batched process\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\n@serve_path_via_http()\n@with_tree(tree={\"bar\": \"bar content\"})\ndef test_annexrepo_fake_dates_disables_batched(sitepath, siteurl, dst):\n    ar = AnnexRepo(dst, create=True, fake_dates=True)\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.add_url_to_file(\"foo-dst\", urljoin(siteurl, \"foo\"), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching addurl call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n    ar.add(\"bar\")\n    ar.commit(\"add bar\")\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.drop_key(ar.get_file_key([\"bar\"]), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching drop_key call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_backends(path):\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, None)\n\n    rmtree(path)\n\n    repo = AnnexRepo(path, backend='MD5E')\n    eq_(repo.default_backends, ['MD5E'])\n\n    # persists\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, ['MD5E'])\n\n\n@skip_ssh\n@with_tempfile\n@with_testrepos('basic_annex', flavors=['local'])\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_annex_ssh(repo_path, remote_1_path, remote_2_path):\n    from datalad import ssh_manager\n    # create remotes:\n    rm1 = AnnexRepo(remote_1_path, create=False)\n    rm2 = AnnexRepo(remote_2_path, create=False)\n\n    # check whether we are the first to use these sockets:\n    socket_1 = opj(ssh_manager.socket_dir, get_connection_hash('datalad-test'))\n    socket_2 = opj(ssh_manager.socket_dir, get_connection_hash('localhost'))\n    datalad_test_was_open = exists(socket_1)\n    localhost_was_open = exists(socket_2)\n\n    # repo to test:AnnexRepo(repo_path)\n    # At first, directly use git to add the remote, which should be recognized\n    # by AnnexRepo's constructor\n    gr = GitRepo(repo_path, create=True)\n    AnnexRepo(repo_path)\n    gr.add_remote(\"ssh-remote-1\", \"ssh://datalad-test\" + remote_1_path)\n\n    # Now, make it an annex:\n    ar = AnnexRepo(repo_path, create=False)\n\n    # connection to 'datalad-test' should be known to ssh manager:\n    assert_in(socket_1, ssh_manager._connections)\n    # but socket was not touched:\n    if datalad_test_was_open:\n        ok_(exists(socket_1))\n    else:\n        ok_(not exists(socket_1))\n\n    from datalad import lgr\n    # remote interaction causes socket to be created:\n    try:\n        # Note: For some reason, it hangs if log_stdout/err True\n        # TODO: Figure out what's going on\n        #  yoh: I think it is because of what is \"TODOed\" within cmd.py --\n        #       trying to log/obtain both through PIPE could lead to lock\n        #       downs.\n        # here we use our swallow_logs to overcome a problem of running under\n        # nosetests without -s, when nose then tries to swallow stdout by\n        # mocking it with StringIO, which is not fully compatible with Popen\n        # which needs its .fileno()\n        with swallow_outputs():\n            ar._run_annex_command('sync',\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_1))\n\n    # add another remote:\n    ar.add_remote('ssh-remote-2', \"ssh://localhost\" + remote_2_path)\n\n    # now, this connection to localhost was requested:\n    assert_in(socket_2, ssh_manager._connections)\n    # but socket was not touched:\n    if localhost_was_open:\n        # FIXME: occasionally(?) fails in V6:\n        if not ar.config.getint(\"annex\", \"version\") == 6:\n            ok_(exists(socket_2))\n    else:\n        ok_(not exists(socket_2))\n\n    # sync with the new remote:\n    try:\n        with swallow_outputs():\n            ar._run_annex_command('sync', annex_options=['ssh-remote-2'],\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_2))\n    ssh_manager.close(ctrl_path=[socket_1, socket_2])\n\n@with_testrepos('basic_annex', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_remove(path1, path2):\n    ar1 = AnnexRepo(path1, create=False)\n    ar2 = AnnexRepo.clone(path1, path2, create=True, direct=True)\n\n    for repo in (ar1, ar2):\n        file_list = repo.get_annexed_files()\n        assert len(file_list) >= 1\n        # remove a single file\n        out = repo.remove(file_list[0])\n        assert_not_in(file_list[0], repo.get_annexed_files())\n        eq_(out[0], file_list[0])\n\n        with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n            f.write(\"whatever\")\n\n        # add it\n        repo.add(\"rm-test.dat\")\n\n        # remove without '--force' should fail, due to staged changes:\n        if repo.is_direct_mode():\n            assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n        else:\n            assert_raises(GitCommandError, repo.remove, \"rm-test.dat\")\n        assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n        # now force:\n        out = repo.remove(\"rm-test.dat\", force=True)\n        assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n        eq_(out[0], \"rm-test.dat\")\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_repo_version(path1, path2, path3):\n    annex = AnnexRepo(path1, create=True, version=6)\n    ok_clean_git(path1, annex=True)\n    version = annex.repo.config_reader().get_value('annex', 'version')\n    eq_(version, 6)\n\n    # default from config item (via env var):\n    with patch.dict('os.environ', {'DATALAD_REPO_VERSION': '6'}):\n        annex = AnnexRepo(path2, create=True)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 6)\n\n        # parameter `version` still has priority over default config:\n        annex = AnnexRepo(path3, create=True, version=5)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 5)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_copy_to(origin, clone):\n    repo = AnnexRepo(origin, create=False)\n    remote = AnnexRepo.clone(origin, clone, create=True)\n    repo.add_remote(\"target\", clone)\n\n    assert_raises(IOError, repo.copy_to, \"doesnt_exist.dat\", \"target\")\n    assert_raises(FileInGitError, repo.copy_to, \"INFO.txt\", \"target\")\n    assert_raises(ValueError, repo.copy_to, \"test-annex.dat\", \"invalid_target\")\n\n    # test-annex.dat has no content to copy yet:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [])\n\n    repo.get(\"test-annex.dat\")\n    # now it has:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [\"test-annex.dat\"])\n    # and will not be copied again since it was already copied\n    eq_(repo.copy_to([\"INFO.txt\", \"test-annex.dat\"], \"target\"), [])\n\n    # Test that if we pass a list of items and annex processes them nicely,\n    # we would obtain a list back. To not stress our tests even more -- let's mock\n    def ok_copy(command, **kwargs):\n        # Check that we do pass to annex call only the list of files which we\n        #  asked to be copied\n        assert_in('copied1', kwargs['annex_options'])\n        assert_in('copied2', kwargs['annex_options'])\n        assert_in('existed', kwargs['annex_options'])\n        return \"\"\"\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied1\"}\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"copied2\"}\n{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey3\", \"file\":\"existed\"}\n\"\"\", \"\"\n    with patch.object(repo, '_run_annex_command', ok_copy):\n        eq_(repo.copy_to([\"copied2\", \"copied1\", \"existed\"], \"target\"),\n            [\"copied1\", \"copied2\"])\n\n    # now let's test that we are correctly raising the exception in case if\n    # git-annex execution fails\n    orig_run = repo._run_annex_command\n\n    # Kinda a bit off the reality since no nonex* would not be returned/handled\n    # by _get_expected_files, so in real life -- wouldn't get report about Incomplete!?\n    def fail_to_copy(command, **kwargs):\n        if command == 'copy':\n            # That is not how annex behaves\n            # http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            # for non-existing files output goes into stderr\n            raise CommandError(\n                \"Failed to run ...\",\n                stdout=\n                    '{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied\"}\\n'\n                    '{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"existed\"}\\n',\n                stderr=\n                    'git-annex: nonex1 not found\\n'\n                    'git-annex: nonex2 not found\\n'\n            )\n        else:\n            return orig_run(command, **kwargs)\n\n    def fail_to_copy_get_expected(files, expr):\n        assert files == [\"copied\", \"existed\", \"nonex1\", \"nonex2\"]\n        return {'akey1': 10}, [\"copied\"]\n\n    with patch.object(repo, '_run_annex_command', fail_to_copy), \\\n            patch.object(repo, '_get_expected_files', fail_to_copy_get_expected):\n        with assert_raises(IncompleteResultsError) as cme:\n            repo.copy_to([\"copied\", \"existed\", \"nonex1\", \"nonex2\"], \"target\")\n    eq_(cme.exception.results, [\"copied\"])\n    eq_(cme.exception.failed, ['nonex1', 'nonex2'])\n\n\n\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_annex_drop(src, dst):\n    ar = AnnexRepo.clone(src, dst)\n    testfile = 'test-annex.dat'\n    assert_false(ar.file_has_content(testfile))\n    ar.get(testfile)\n    ok_(ar.file_has_content(testfile))\n\n    # drop file by name:\n    result = ar.drop([testfile])\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['file'], testfile)\n\n    ar.get(testfile)\n\n    # drop file by key:\n    testkey = ar.get_file_key(testfile)\n    result = ar.drop([testkey], key=True)\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['key'], testkey)\n\n    # insufficient arguments:\n    assert_raises(TypeError, ar.drop)\n    assert_raises(InsufficientArgumentsError, ar.drop, [], options=[\"--jobs=5\"])\n    assert_raises(InsufficientArgumentsError, ar.drop, [])\n\n    # too much arguments:\n    assert_raises(CommandError, ar.drop, ['.'], options=['--all'])\n\n\n@with_tree({\"a.txt\": \"a\", \"b.txt\": \"b\", \"c.py\": \"c\", \"d\": \"d\"})\ndef test_annex_get_annexed_files(path):\n    repo = AnnexRepo(path)\n    repo.add(\".\")\n    repo.commit()\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n\n    repo.drop(\"a.txt\", options=[\"--force\"])\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        {\"b.txt\", \"c.py\", \"d\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\"])),\n        {\"a.txt\", \"b.txt\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True,\n                                   patterns=[\"*.txt\"])),\n        {\"b.txt\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\", \"*.py\"])),\n        {\"a.txt\", \"b.txt\", \"c.py\"})\n\n    eq_(set(repo.get_annexed_files()),\n        set(repo.get_annexed_files(patterns=[\"*\"])))\n\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        set(repo.get_annexed_files(with_content_only=True, patterns=[\"*\"])))\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_annex_remove(path):\n    repo = AnnexRepo(path, create=False)\n\n    file_list = repo.get_annexed_files()\n    assert len(file_list) >= 1\n    # remove a single file\n    out = repo.remove(file_list[0])\n    assert_not_in(file_list[0], repo.get_annexed_files())\n    eq_(out[0], file_list[0])\n\n    with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    # add it\n    repo.add(\"rm-test.dat\")\n\n    # remove without '--force' should fail, due to staged changes:\n    assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n    assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n    # now force:\n    out = repo.remove(\"rm-test.dat\", force=True)\n    assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n    eq_(out[0], \"rm-test.dat\")\n\n\n@with_batch_direct\n@with_testrepos('basic_annex', flavors=['clone'], count=1)\ndef test_is_available(batch, direct, p):\n    annex = AnnexRepo(p)\n\n    # bkw = {'batch': batch}\n    if batch:\n        is_available = partial(annex.is_available, batch=batch)\n    else:\n        is_available = annex.is_available\n\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n\n    # explicit is to verify data type etc\n    assert is_available(key, key=True) is True\n    assert is_available(fname) is True\n\n    # known remote but doesn't have it\n    assert is_available(fname, remote='origin') is False\n    # it is on the 'web'\n    assert is_available(fname, remote='web') is True\n    # not effective somehow :-/  may be the process already running or smth\n    # with swallow_logs(), swallow_outputs():  # it will complain!\n    assert is_available(fname, remote='unknown') is False\n    assert_false(is_available(\"boguskey\", key=True))\n\n    # remove url\n    urls = annex.get_urls(fname) #, **bkw)\n    assert(len(urls) == 1)\n    annex.rm_url(fname, urls[0])\n\n    assert is_available(key, key=True) is False\n    assert is_available(fname) is False\n    assert is_available(fname, remote='web') is False\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_add_no_dotfiles(path):\n    ar = AnnexRepo(path, create=True)\n    print(ar.path)\n    assert_true(os.path.exists(ar.path))\n    assert_false(ar.dirty)\n    os.makedirs(opj(ar.path, '.datalad'))\n    # we don't care about empty directories\n    assert_false(ar.dirty)\n    with open(opj(ar.path, '.datalad', 'somefile'), 'w') as f:\n        f.write('some content')\n    # make sure the repo is considered dirty now\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # no file is being added, as dotfiles/directories are ignored by default\n    ar.add('.', git=False)\n    # double check, still dirty\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # now add to git, and it should work\n    ar.add('.', git=True)\n    # all in index\n    assert_true(ar.dirty)\n    # TODO: has been more specific:\n    # assert_false(ar.repo.is_dirty(\n    #     index=False, working_tree=True, untracked_files=True, submodules=True))\n    ar.commit(msg=\"some\")\n    # all committed\n    assert_false(ar.dirty)\n    # not known to annex\n    assert_false(ar.is_under_annex(opj(ar.path, '.datalad', 'somefile')))\n\n\n@with_tempfile\ndef test_annex_version_handling(path):\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n         patch.object(AnnexRepo, '_check_git_annex_version',\n                      auto_spec=True,\n                      side_effect=AnnexRepo._check_git_annex_version) \\\n            as cmpc, \\\n         patch.object(external_versions, '_versions',\n                      {'cmd:annex': AnnexRepo.GIT_ANNEX_MIN_VERSION}):\n            eq_(AnnexRepo.git_annex_version, None)\n            ar1 = AnnexRepo(path, create=True)\n            assert(ar1)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n            # 2nd time must not be called\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            ar2 = AnnexRepo(path)\n            assert(ar2)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n            patch.object(AnnexRepo, '_check_git_annex_version',\n                         auto_spec=True,\n                         side_effect=AnnexRepo._check_git_annex_version):\n        # no git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': None}):\n            eq_(AnnexRepo.git_annex_version, None)\n            with assert_raises(MissingExternalDependency) as cme:\n                try:\n                    # Note: Remove to cause creation of a new instance\n                    rmtree(path)\n                except OSError:\n                    pass\n                AnnexRepo(path)\n            if linux_distribution_name == 'debian':\n                assert_in(\"http://neuro.debian.net\", str(cme.exception))\n            eq_(AnnexRepo.git_annex_version, None)\n\n        # outdated git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': '6.20160505'}):\n            eq_(AnnexRepo.git_annex_version, None)\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n            # and we don't assign it\n            eq_(AnnexRepo.git_annex_version, None)\n            # so we could still fail\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n\n\ndef test_ProcessAnnexProgressIndicators():\n    irrelevant_lines = (\n        'abra',\n        '{\"some_json\": \"sure thing\"}'\n    )\n    # regular lines, without completion for known downloads\n    success_lines = (\n        '{\"command\":\"get\",\"note\":\"\",\"success\":true,\"key\":\"key1\",\"file\":\"file1\"}',\n        '{\"command\":\"comm\",\"note\":\"\",\"success\":true,\"key\":\"backend-s10--key2\"}',\n    )\n    progress_lines = (\n        '{\"byte-progress\":10,\"action\":{\"command\":\"get\",\"note\":\"from web...\",'\n            '\"key\":\"key1\",\"file\":\"file1\"},\"percent-progress\":\"10%\"}',\n    )\n\n    # without providing expected entries\n    proc = ProcessAnnexProgressIndicators()\n    # when without any target downloads, there is no total_pbar\n    assert_is(proc.total_pbar, None)\n    # for regular lines -- should just return them without side-effects\n    for l in irrelevant_lines + success_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    eq_(proc(progress_lines[0]), None)\n    eq_(len(proc.pbars), 1)\n    # but when we finish download -- should get cleared\n    eq_(proc(success_lines[0]), success_lines[0])\n    eq_(proc.pbars, {})\n    # and no side-effect of any kind in finish\n    eq_(proc.finish(), None)\n\n    proc = ProcessAnnexProgressIndicators(expected={'key1': 100, 'key2': None})\n    # when without any target downloads, there is no total_pbar\n    assert(proc.total_pbar is not None)\n    eq_(proc.total_pbar.total, 100)  # as much as it knows at this point\n    eq_(proc.total_pbar.current, 0)\n    # for regular lines -- should still just return them without side-effects\n    for l in irrelevant_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    # it doesn't swallow everything -- so there will be side-effects in output\n    with swallow_outputs() as cmo:\n        eq_(proc(progress_lines[0]), None)\n        eq_(len(proc.pbars), 1)\n        # but when we finish download -- should get cleared\n        eq_(proc(success_lines[0]), success_lines[0])\n        eq_(proc.pbars, {})\n        out = cmo.out\n\n    from datalad.ui import ui\n    from datalad.ui.dialog import SilentConsoleLog\n\n    assert out \\\n        if not isinstance(ui.ui, SilentConsoleLog) else not out\n    assert proc.total_pbar is not None\n    # and no side-effect of any kind in finish\n    with swallow_outputs() as cmo:\n        eq_(proc.finish(), None)\n        eq_(proc.total_pbar, None)\n\n\n@with_tempfile\n@with_tempfile\ndef test_get_description(path1, path2):\n    annex1 = AnnexRepo(path1, create=True)\n    # some content for git-annex branch\n    create_tree(path1, {'1.dat': 'content'})\n    annex1.add('1.dat', git=False)\n    annex1.commit(\"msg\")\n    annex1_description = annex1.get_description()\n    assert_not_equal(annex1_description, path1)\n\n    annex2 = AnnexRepo(path2, create=True, description='custom 2')\n    eq_(annex2.get_description(), 'custom 2')\n    # not yet known\n    eq_(annex2.get_description(uuid=annex1.uuid), None)\n\n    annex2.add_remote('annex1', path1)\n    annex2.fetch('annex1')\n    # it will match the remote name\n    eq_(annex2.get_description(uuid=annex1.uuid),\n        annex1_description + ' [annex1]')\n    # add a little probe file to make sure it stays untracked\n    create_tree(path1, {'probe': 'probe'})\n    assert_not_in('probe', annex2.get_indexed_files())\n    annex2.merge_annex('annex1')\n    assert_not_in('probe', annex2.get_indexed_files())\n    # but let's remove the remote\n    annex2.remove_remote('annex1')\n    eq_(annex2.get_description(uuid=annex1.uuid), annex1_description)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_flyweight(path1, path2):\n\n    repo1 = AnnexRepo(path1, create=True)\n    assert_is_instance(repo1, AnnexRepo)\n    # instantiate again:\n    repo2 = AnnexRepo(path1, create=False)\n    assert_is_instance(repo2, AnnexRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in an different way:\n    with chpwd(path1):\n        repo3 = AnnexRepo(relpath(path1, start=path2), create=False)\n        assert_is_instance(repo3, AnnexRepo)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # but path attribute is absolute, so they are still equal:\n    ok_(repo1 == repo3)\n\n    # Now, let's try to get a GitRepo instance from a path, we already have an\n    # AnnexRepo of\n    repo4 = GitRepo(path1)\n    assert_is_instance(repo4, GitRepo)\n    assert_not_is_instance(repo4, AnnexRepo)\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_AnnexRepo_get_toppath(repo, tempdir, repo2):\n\n    reporeal = realpath(repo)\n    eq_(AnnexRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(AnnexRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    AnnexRepo(repo2, create=True)\n    repo2real = realpath(repo2)\n    nested = opj(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(AnnexRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(AnnexRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(AnnexRepo.get_toppath(tempdir), None)\n\n\n@with_testrepos(\".*basic.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_add_submodule(source, path):\n\n    top_repo = AnnexRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n\n    ok_clean_git(top_repo, annex=True)\n    ok_clean_git(opj(path, 'sub'), annex=False)\n\n\ndef test_AnnexRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@known_failure_v6  #FIXME\ndef test_AnnexRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_dirty(path):\n\n    repo = AnnexRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # pure git operations:\n    # untracked file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt', git=True)\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    if not repo.config.getint(\"annex\", \"version\") == 6:\n        ok_(not repo.dirty)\n    # modified file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt', git=True)\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # annex operations:\n    # untracked file\n    with open(opj(path, 'file2.txt'), 'w') as f:\n        f.write('different content')\n    ok_(repo.dirty)\n    # annexed file\n    repo.add('file2.txt', git=False)\n    if not repo.is_direct_mode():\n        # in direct mode 'annex add' results in a clean repo\n        ok_(repo.dirty)\n        # commit\n        repo.commit(\"file2.txt annexed\")\n    ok_(not repo.dirty)\n\n    # TODO: unlock/modify\n\n    # TODO: submodules\n\n\ndef _test_status(ar):\n    # TODO: plain git submodule and even deeper hierarchy?\n    #       => complete recursion to work if started from within plain git;\n    #       But this is then relevant for Dataset.status() - not herein\n\n    def sync_wrapper(push=False, pull=False, commit=False):\n        # wraps common annex-sync call, since it currently fails under\n        # mysterious circumstances in V6 adjusted branch setups\n        try:\n            ar.sync(push=push, pull=pull, commit=commit)\n        except CommandError as e:\n            if \"fatal: entry 'submod' object type (blob) doesn't match mode type \" \\\n               \"(commit)\" in e.stderr:\n                # some bug in adjusted branch(?) + submodule\n                # TODO: figure out and probably report\n                # stdout:\n                # commit\n                # [adjusted/master(unlocked) ae3e9a7] git-annex in ben@tree:/tmp/datalad_temp_test_AnnexRepo_status7qKhRQ\n                #  4 files changed, 4 insertions(+), 2 deletions(-)\n                #  create mode 100644 fifth\n                #  create mode 100644 sub/third\n                # ok\n                #\n                # failed\n                # stderr:\n                # fatal: entry 'submod' object type (blob) doesn't match mode type (commit)\n                # git-annex: user error (git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"mktree\",\"--batch\",\"-z\"] exited 128)\n                # git-annex: sync: 1 failed\n\n                # But it almost works - so apperently nothing to do\n                import logging\n                lgr = logging.getLogger(\"datalad.support.tests.test-status\")\n                lgr.warning(\"DEBUG: v6 sync failure\")\n\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n\n    # untracked files:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"it's huge!\")\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"looser\")\n\n    stat['untracked'].append('first')\n    stat['untracked'].append('second')\n    eq_(stat, ar.get_status())\n\n    # add a file to git\n    ar.add('first', git=True)\n    sync_wrapper()\n    stat['untracked'].remove('first')\n    stat['added'].append('first')\n    eq_(stat, ar.get_status())\n\n    # add a file to annex\n    ar.add('second')\n    sync_wrapper()\n    stat['untracked'].remove('second')\n    if not ar.is_direct_mode():\n        # in direct mode annex-status doesn't report an added file 'added'\n        stat['added'].append('second')\n    eq_(stat, ar.get_status())\n\n    # commit to be clean again:\n    ar.commit(\"added first and second\")\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n    # create a file to be unannexed:\n    with open(opj(ar.path, 'fifth'), 'w') as f:\n        f.write(\"total disaster\")\n\n    ar.add('fifth')\n    sync_wrapper()\n    # TODO:\n    # Note: For some reason this seems to be the only place, where we actually\n    # need to call commit via annex-proxy. If called via '-c core.bare=False'\n    # and/or '--work-tree=.' the file ends up in git instead of annex.\n    # Note 2: This is only if we explicitly pass a path. Otherwise it works\n    # without annex-proxy.\n    ar.commit(msg=\"fifth to be unannexed\", files='fifth',\n              proxy=ar.is_direct_mode())\n    eq_(stat, ar.get_status())\n\n    ar.unannex('fifth')\n\n    sync_wrapper(pull=False, push=False, commit=True)\n    stat['untracked'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # modify a file in git:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"increased tremendousness\")\n    stat['modified'].append('first')\n    eq_(stat, ar.get_status())\n\n    # modify an annexed file:\n    if not ar.is_direct_mode():\n        # actually: if 'second' isn't locked, which is the case in direct mode\n        ar.unlock('second')\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].append('second')\n        eq_(stat, ar.get_status())\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"Needed to unlock first. Sad!\")\n    if not ar.is_direct_mode():\n        ar.add('second')  # => modified\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].remove('second')\n    stat['modified'].append('second')\n    sync_wrapper()\n    eq_(stat, ar.get_status())\n\n    # create something in a subdir\n    os.mkdir(opj(ar.path, 'sub'))\n    with open(opj(ar.path, 'sub', 'third'), 'w') as f:\n        f.write(\"tired of winning\")\n\n    # Note, that this is different from 'git status',\n    # which would just say 'sub/':\n    stat['untracked'].append(opj('sub', 'third'))\n    eq_(stat, ar.get_status())\n\n    # test parameters for status to restrict results:\n    # limit requested states:\n    limited_status = ar.get_status(untracked=True, deleted=False, modified=True,\n                                   added=True, type_changed=False)\n    eq_(len(limited_status), 3)\n    ok_(all([k in ('untracked', 'modified', 'added') for k in limited_status]))\n    eq_(stat['untracked'], limited_status['untracked'])\n    eq_(stat['modified'], limited_status['modified'])\n    eq_(stat['added'], limited_status['added'])\n    # limit requested files:\n    limited_status = ar.get_status(path=opj('sub', 'third'))\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    ok_(all([len(limited_status[l]) == 0 for l in ('modified', 'added',\n                                                   'deleted', 'type_changed')]))\n    # again, with a list:\n    limited_status = ar.get_status(path=[opj('sub', 'third'), 'second'])\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    eq_(limited_status['modified'], ['second'])\n    ok_(all([len(limited_status[l]) == 0 for l in ('added', 'deleted',\n                                                   'type_changed')]))\n\n    # create a subrepo:\n    sub = AnnexRepo(opj(ar.path, 'submod'), create=True)\n    # nothing changed, it's empty besides .git, which is ignored\n    eq_(stat, ar.get_status())\n\n    # file in subrepo\n    with open(opj(ar.path, 'submod', 'fourth'), 'w') as f:\n        f.write(\"this is a birth certificate\")\n    stat['untracked'].append(opj('submod', 'fourth'))\n    eq_(stat, ar.get_status())\n\n    # add to subrepo\n    sub.add('fourth')\n    sub.commit(msg=\"birther mod init'ed\")\n    stat['untracked'].remove(opj('submod', 'fourth'))\n\n    if ar.get_active_branch().endswith('(unlocked)') and \\\n       'adjusted' in ar.get_active_branch():\n        # we are running on adjusted branch => do it in submodule, too\n        sub.adjust()\n\n    # Note, that now the non-empty repo is untracked\n    stat['untracked'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the submodule\n    ar.add_submodule('submod', url=opj(curdir, 'submod'))\n\n    stat['untracked'].remove('submod/')\n    stat['added'].append('.gitmodules')\n\n    # 'submod/' might either be reported as 'added' or 'modified'.\n    # Therefore more complex assertions at this point:\n    reported_stat = ar.get_status()\n    eq_(stat['untracked'], reported_stat['untracked'])\n    eq_(stat['deleted'], reported_stat['deleted'])\n    eq_(stat['type_changed'], reported_stat['type_changed'])\n    ok_(stat['added'] == reported_stat['added'] or\n        stat['added'] + ['submod/'] == reported_stat['added'])\n    ok_(stat['modified'] == reported_stat['modified'] or\n        stat['modified'] + ['submod/'] == reported_stat['modified'])\n\n    # simpler assertion if we ignore submodules:\n    eq_(stat, ar.get_status(submodules=False))\n\n    # commit the submodule\n    # in direct mode, commit of a removed submodule fails with:\n    #  error: unable to index file submod\n    #  fatal: updating files failed\n    #\n    # - this happens, when commit is called with -c core.bare=False\n    # - it works when called via annex proxy\n    # - if we add a submodule instead of removing one, it's vice versa with\n    #   the very same error message\n\n    ar.commit(msg=\"submodule added\", files=['.gitmodules', 'submod'])\n\n    stat['added'].remove('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # add another file to submodule\n    with open(opj(ar.path, 'submod', 'not_tracked'), 'w') as f:\n        f.write(\"#LastNightInSweden\")\n    stat['modified'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the untracked file:\n    sub.add('not_tracked')\n    if sub.is_direct_mode():\n        # 'sub' is now considered to be clean; therefore it's not reported as\n        # modified upwards\n        # This is consistent in a way, but surprising in another ...\n        pass\n    else:\n        eq_(stat, ar.get_status())\n    sub.commit(msg=\"added file not_tracked\")\n    # 'submod/' still modified when looked at from above:\n    eq_(stat, ar.get_status())\n\n    ar.add('submod', git=True)\n    ar.commit(msg=\"submodule modified\", files='submod')\n    stat['modified'].remove('submod/')\n    eq_(stat, ar.get_status())\n\n    # clean again:\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n\n    ar.add('first', git=True)\n    ar.add(opj('sub', 'third'))\n    ar.add('fifth')\n    sync_wrapper()\n\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # mixed annexed/not-annexed files ATm can't be committed with explicitly\n        # given paths in v6\n        # See:\n        # http://git-annex.branchable.com/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6\n        ar.commit()\n    else:\n        super(AnnexRepo, ar).commit(files=['first', 'fifth', opj('sub', 'third'), 'second'])\n    eq_(stat, ar.get_status())\n\n    # remove a file in annex:\n    ar.remove('fifth')\n    stat['deleted'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # remove a file in git:\n    ar.remove('first')\n    stat['deleted'].append('first')\n    eq_(stat, ar.get_status())\n\n    # remove a submodule:\n    # rm; git rm; git commit\n    sub.precommit()  # Do precommit so there are not active batched processes etc\n    from datalad.utils import rmtree\n    rmtree(opj(ar.path, 'submod'))\n    stat['deleted'].append('submod')\n    eq_(stat, ar.get_status())\n    # recreate an empty mountpoint, since we currently do it in uninstall:\n    os.makedirs(opj(ar.path, 'submod'))\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n    ar.remove('submod')\n    # TODO: Why the difference?\n    stat['deleted'].append('submod')\n    stat['modified'].append('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # Note: Here again we need to use annex-proxy; This contradicts the addition\n    # of the very same submodule, which we needed to commit via\n    # -c core.bare=False instead. Otherwise the very same failure happens.\n    # Just vice versa. See above where 'submod' is added.\n    ar.commit(\"submod removed\", files=['submod', '.gitmodules'])\n    stat['modified'].remove('.gitmodules')\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_status(path, path2):\n\n    ar = AnnexRepo(path, create=True)\n    _test_status(ar)\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # in case of v6 have a second run with adjusted branch feature:\n        ar2 = AnnexRepo(path2, create=True)\n        ar2.commit(msg=\"empty commit to create branch 'master'\",\n                   options=['--allow-empty'])\n        ar2._run_annex_command('adjust', annex_options=['--unlock'])\n        _test_status(ar2)\n\n\n\n# TODO: test dirty\n# TODO: GitRep.dirty\n# TODO: test/utils ok_clean_git\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_set_remote_url(path):\n\n    ar = AnnexRepo(path, create=True)\n    ar.add_remote('some', 'http://example.com/.git')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://example.com/.git')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n    # change url:\n    ar.set_remote_url('some', 'http://believe.it')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://believe.it')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n\n    # set push url:\n    ar.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    assert_equal(ar.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n    assert_in('remote.some.annexurl', ar.config.keys())\n    assert_equal(ar.config['remote.some.annexurl'],\n                 'ssh://whatever.ru')\n\n\n@with_tempfile(mkdir=True)\ndef test_wanted(path):\n    ar = AnnexRepo(path, create=True)\n    eq_(ar.get_preferred_content('wanted'), None)\n    # test samples with increasing \"trickiness\"\n    for v in (\"standard\",\n              \"include=*.nii.gz or include=*.nii\",\n              \"exclude=archive/* and (include=*.dat or smallerthan=2b)\"\n              ):\n        ar.set_preferred_content('wanted', expr=v)\n        eq_(ar.get_preferred_content('wanted'), v)\n    # give it some file so clone/checkout works without hiccups\n    create_tree(ar.path, {'1.dat': 'content'})\n    ar.add('1.dat')\n    ar.commit(msg=\"blah\")\n    # make a clone and see if all cool there\n    # intentionally clone as pure Git and do not annex init so to see if we\n    # are ignoring crummy log msgs\n    ar1_path = ar.path + '_1'\n    GitRepo.clone(ar.path, ar1_path)\n    ar1 = AnnexRepo(ar1_path, init=False)\n    eq_(ar1.get_preferred_content('wanted'), None)\n    eq_(ar1.get_preferred_content('wanted', 'origin'), v)\n    ar1.set_preferred_content('wanted', expr='standard')\n    eq_(ar1.get_preferred_content('wanted'), 'standard')\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_metadata(path):\n    # prelude\n    ar = AnnexRepo(path, create=True)\n    create_tree(\n        path,\n        {\n            'up.dat': 'content',\n            'd o\"w n': {\n                'd o w n.dat': 'lowcontent'\n            }\n        })\n    ar.add('.', git=False)\n    ar.commit('content')\n    ok_clean_git(path)\n    # fugue\n    # doesn't do anything if there is nothing to do\n    ar.set_metadata('up.dat')\n    eq_([], list(ar.get_metadata(None)))\n    eq_([], list(ar.get_metadata('')))\n    eq_([], list(ar.get_metadata([])))\n    eq_({'up.dat': {}}, dict(ar.get_metadata('up.dat')))\n    # basic invocation\n    eq_(1, len(list(ar.set_metadata(\n        'up.dat',\n        reset={'mike': 'awesome'},\n        add={'tag': 'awesome'},\n        remove={'tag': 'awesome'},  # cancels prev, just to use it\n        init={'virgin': 'true'},\n        purge=['nothere']))))\n    # no timestamps by default\n    md = dict(ar.get_metadata('up.dat'))\n    deq_({'up.dat': {\n        'virgin': ['true'],\n        'mike': ['awesome']}},\n        md)\n    # matching timestamp entries for all keys\n    md_ts = dict(ar.get_metadata('up.dat', timestamps=True))\n    for k in md['up.dat']:\n        assert_in('{}-lastchanged'.format(k), md_ts['up.dat'])\n    assert_in('lastchanged', md_ts['up.dat'])\n    # recursive needs a flag\n    assert_raises(CommandError, list, ar.set_metadata('.', purge=['virgin']))\n    list(ar.set_metadata('.', purge=['virgin'], recursive=True))\n    deq_({'up.dat': {\n        'mike': ['awesome']}},\n        dict(ar.get_metadata('up.dat')))\n    # Use trickier tags (spaces, =)\n    list(ar.set_metadata('.', reset={'tag': 'one and= '}, purge=['mike'], recursive=True))\n    playfile = opj('d o\"w n', 'd o w n.dat')\n    target = {\n        'up.dat': {\n            'tag': ['one and= ']},\n        playfile: {\n            'tag': ['one and= ']}}\n    deq_(target, dict(ar.get_metadata('.')))\n    # incremental work like a set\n    list(ar.set_metadata(playfile, add={'tag': 'one and= '}))\n    deq_(target, dict(ar.get_metadata('.')))\n    list(ar.set_metadata(playfile, add={'tag': ' two'}))\n    # returned values are sorted\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    # init honor prior values\n    list(ar.set_metadata(playfile, init={'tag': 'three'}))\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    list(ar.set_metadata(playfile, remove={'tag': ' two'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # remove non-existing doesn't error and doesn't change anything\n    list(ar.set_metadata(playfile, remove={'ether': 'best'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # add works without prior existence\n    list(ar.set_metadata(playfile, add={'novel': 'best'}))\n    eq_(['best'], dict(ar.get_metadata(playfile))[playfile]['novel'])\n\n\n@with_tree(tree={'file.txt': 'content'})\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_batched_and_set_metadata(path, url, dest):\n    ar = AnnexRepo(dest, create=True)\n    fname = \"file.txt\"\n    ar.add_url_to_file(fname, urljoin(url, fname), batch=True)\n    list(ar.set_metadata(fname, init={\"number\": \"one\"}))\n    eq_([\"one\"], dict(ar.get_metadata(fname))[fname][\"number\"])\n\n\n@with_tempfile(mkdir=True)\ndef test_change_description(path):\n    # prelude\n    ar = AnnexRepo(path, create=True, description='some')\n    eq_(ar.get_description(), 'some')\n    # try change it\n    ar = AnnexRepo(path, create=False, init=True, description='someother')\n    # this doesn't cut the mustard, still old\n    eq_(ar.get_description(), 'some')\n    # need to resort to \"internal\" helper\n    ar._init(description='someother')\n    eq_(ar.get_description(), 'someother')\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_corresponding_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we should be on master or a corresponding branch like annex/direct/master\n    # respectively if ran in direct mode build.\n    # We want to get 'master' in any case\n    eq_('master', ar.get_corresponding_branch())\n\n    # special case v6 adjusted branch is not provided by a dedicated build:\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        # as above, we still want to get 'master', while being on\n        # 'adjusted/master(unlocked)'\n        eq_('adjusted/master(unlocked)', ar.get_active_branch())\n        eq_('master', ar.get_corresponding_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_tracking_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we want the relation to original branch, especially in direct mode\n    # or even in v6 adjusted branch\n    eq_(('origin', 'refs/heads/master'), ar.get_tracking_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_is_managed_branch(path):\n\n    ar = AnnexRepo(path)\n\n    if ar.is_direct_mode():\n        ok_(ar.is_managed_branch())\n    else:\n        # ATM only direct mode and v6 adjusted branches should return True.\n        # Adjusted branch requires a call of git-annex-adjust and shouldn't\n        # be the state of a fresh clone\n        ok_(not ar.is_managed_branch())\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        ok_(ar.is_managed_branch())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_AnnexRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(AnnexRepo, path, store)\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_is_not_special(path):\n    ar = AnnexRepo(path, create=True)\n    # doesn't exist -- we fail by default\n    assert_raises(RemoteNotAvailableError, ar.is_special_annex_remote, \"fake\")\n    assert_false(ar.is_special_annex_remote(\"fake\", check_if_known=False))\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_dates(path):\n    ar = AnnexRepo(path, create=True, fake_dates=True)\n    timestamp = ar.config.obtain(\"datalad.fake-dates-start\") + 1\n    # Commits from the \"git annex init\" call are one second ahead.\n    for commit in ar.get_branch_commits(\"git-annex\"):\n        eq_(timestamp, commit.committed_date)\n    assert_in(\"timestamp={}s\".format(timestamp),\n              ar.repo.git.cat_file(\"blob\", \"git-annex:uuid.log\"))\n\n\ndef test_get_size_from_perc_complete():\n    f = _get_size_from_perc_complete\n    eq_(f(0, 0), 0)\n    eq_(f(0, '0'), 0)\n    eq_(f(100, '0'), 0)  # we do not know better\n    eq_(f(1, '1'), 100)\n\n\n# to prevent regression\n# http://git-annex.branchable.com/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/\n# It is disabled because is not per se relevant to DataLad since we do not\n# Since we invoke from the top of the repo, we do not hit it,\n# but thought to leave it around if we want to enforce/test system-wide git being\n# compatible with annex for v6 mode\n@with_tempfile(mkdir=True)\ndef _test_add_under_subdir(path):\n    ar = AnnexRepo(path, create=True, version=6)\n    gr = GitRepo(path)  # \"Git\" view over the repository, so we force \"git add\"\n    subdir = opj(path, 'sub')\n    subfile = opj('sub', 'empty')\n    # os.mkdir(subdir)\n    create_tree(subdir, {'empty': ''})\n    runner = Runner(cwd=subdir)\n    with chpwd(subdir):\n        runner(['git', 'add', 'empty'])  # should add sucesfully\n        # gr.commit('important') #\n        runner(['git', 'commit', '-m', 'important'])\n        ar.is_under_annex(subfile)",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class AnnexRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import known_failure_v6\n\nimport logging\nfrom functools import partial\nimport os\nfrom os import mkdir\nfrom os.path import join as opj\nfrom os.path import basename\nfrom os.path import realpath\nfrom os.path import relpath\nfrom os.path import curdir\nfrom os.path import pardir\nfrom os.path import exists\nfrom shutil import copyfile\nfrom nose.tools import assert_not_is_instance\n\nfrom six.moves.urllib.parse import urljoin\nfrom six.moves.urllib.parse import urlsplit\n\nimport git\nfrom git import GitCommandError\nfrom mock import patch\nimport gc\n\nfrom datalad.cmd import Runner\n\nfrom datalad.support.external_versions import external_versions\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.utils import on_windows\nfrom datalad.utils import chpwd\nfrom datalad.utils import rmtree\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import unlink\n\nfrom datalad.tests.utils import ignore_nose_capturing_stdout\nfrom datalad.tests.utils import assert_cwd_unchanged\nfrom datalad.tests.utils import with_testrepos\nfrom datalad.tests.utils import with_tempfile\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import with_batch_direct\nfrom datalad.tests.utils import assert_dict_equal as deq_\nfrom datalad.tests.utils import assert_is_instance\nfrom datalad.tests.utils import assert_false\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_is\nfrom datalad.tests.utils import assert_not_in\nfrom datalad.tests.utils import assert_re_in\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import assert_not_equal\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_true\nfrom datalad.tests.utils import eq_\nfrom datalad.tests.utils import ok_\nfrom datalad.tests.utils import ok_git_config_not_empty\nfrom datalad.tests.utils import ok_annex_get\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_file_has_content\nfrom datalad.tests.utils import swallow_logs\nfrom datalad.tests.utils import swallow_outputs\nfrom datalad.tests.utils import local_testrepo_flavors\nfrom datalad.tests.utils import serve_path_via_http\nfrom datalad.tests.utils import get_most_obscure_supported_name\nfrom datalad.tests.utils import OBSCURE_FILENAME\nfrom datalad.tests.utils import SkipTest\nfrom datalad.tests.utils import skip_ssh\nfrom datalad.tests.utils import find_files\n\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.exceptions import CommandNotAvailableError\nfrom datalad.support.exceptions import FileNotInRepositoryError\nfrom datalad.support.exceptions import FileNotInAnnexError\nfrom datalad.support.exceptions import FileInGitError\nfrom datalad.support.exceptions import OutOfSpaceError\nfrom datalad.support.exceptions import RemoteNotAvailableError\nfrom datalad.support.exceptions import OutdatedExternalDependency\nfrom datalad.support.exceptions import MissingExternalDependency\nfrom datalad.support.exceptions import InsufficientArgumentsError\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.exceptions import IncompleteResultsError\n\nfrom datalad.support.gitrepo import GitRepo\n\n# imports from same module:\nfrom datalad.support.annexrepo import (\n    AnnexRepo,\n    ProcessAnnexProgressIndicators,\n    _get_size_from_perc_complete,\n)\nfrom .utils import check_repo_deals_with_inode_change\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_instance_from_clone(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(dst, '.git', 'annex')))\n\n    # do it again should raise GitCommandError since git will notice\n    # there's already a git-repo at that path and therefore can't clone to `dst`\n    with swallow_logs(new_level=logging.WARN) as cm:\n        assert_raises(GitCommandError, AnnexRepo.clone, src, dst)\n        if git.__version__ != \"1.0.2\" and git.__version__ != \"2.0.5\":\n            assert(\"already exists\" in cm.out)\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_instance_from_existing(path):\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_instance_brand_new(path):\n\n    GitRepo(path)\n    assert_raises(RuntimeError, AnnexRepo, path, create=False)\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_crippled_filesystem(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    # fake git-annex entries in .git/config:\n    writer = ar.repo.config_writer()\n    writer.set_value(\"annex\", \"crippledfilesystem\", True)\n    writer.release()\n    ok_(ar.is_crippled_fs())\n    writer.set_value(\"annex\", \"crippledfilesystem\", False)\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n    # since we can't remove the entry, just rename it to fake its absence:\n    writer.rename_section(\"annex\", \"removed\")\n    writer.set_value(\"annex\", \"something\", \"value\")\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_is_direct_mode(path):\n\n    ar = AnnexRepo(path)\n    eq_(ar.config.getbool(\"annex\", \"direct\", False),\n        ar.is_direct_mode())\n\n\n@with_tempfile()\ndef test_AnnexRepo_is_direct_mode_gitrepo(path):\n    repo = GitRepo(path, create=True)\n    # artificially make .git/annex so no annex section gets initialized\n    # in .git/config.  We did manage somehow to make this happen (via publish)\n    # but didn't reproduce yet, so just creating manually\n    mkdir(opj(repo.path, '.git', 'annex'))\n    ar = AnnexRepo(path, init=False, create=False)\n    # It is unlikely though that annex would be in direct mode (requires explicit)\n    # annex magic, without having annex section under .git/config\n    dm = ar.is_direct_mode()\n\n    if ar.is_crippled_fs() or on_windows:\n        ok_(dm)\n    else:\n        assert_false(dm)\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_set_direct_mode(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available:\n        assert_raises(CommandError, ar.set_direct_mode, True)\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n\n    ar.set_direct_mode(True)\n    ok_(ar.is_direct_mode(), \"Switching to direct mode failed.\")\n    if ar.is_crippled_fs():\n        assert_raises(CommandNotAvailableError, ar.set_direct_mode, False)\n        ok_(\n            ar.is_direct_mode(),\n            \"Indirect mode on crippled fs detected. Shouldn't be possible.\")\n    else:\n        ar.set_direct_mode(False)\n        assert_false(ar.is_direct_mode(), \"Switching to indirect mode failed.\")\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_annex_proxy(src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path)\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available and therefore no 'annex proxy':\n        assert_raises(CommandError, ar.proxy, ['git', 'status'])\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n    ar.set_direct_mode(True)\n\n    # annex proxy raises in indirect mode:\n    try:\n        ar.set_direct_mode(False)\n        assert_raises(CommandNotAvailableError, ar.proxy, ['git', 'status'])\n    except CommandNotAvailableError:\n        # we can't switch to indirect\n        pass\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_key(src, annex_path):\n\n    ar = AnnexRepo.clone(src, annex_path)\n\n    # test-annex.dat should return the correct key:\n    eq_(\n        ar.get_file_key(\"test-annex.dat\"),\n        'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat')\n\n    # and should take a list with an empty string as result, if a file wasn't\n    # in annex:\n    eq_(\n        ar.get_file_key([\"filenotpresent.wtf\", \"test-annex.dat\"]),\n        ['', 'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat']\n    )\n\n    # test.dat is actually in git\n    # should raise Exception; also test for polymorphism\n    assert_raises(IOError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileNotInAnnexError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileInGitError, ar.get_file_key, \"test.dat\")\n\n    # filenotpresent.wtf doesn't even exist\n    assert_raises(IOError, ar.get_file_key, \"filenotpresent.wtf\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_get_outofspace(annex_path):\n    ar = AnnexRepo(annex_path, create=True)\n\n    def raise_cmderror(*args, **kwargs):\n        raise CommandError(\n            cmd=\"whatever\",\n            stderr=\"junk around not enough free space, need 905.6 MB more and after\"\n        )\n\n    with patch.object(AnnexRepo, '_run_annex_command', raise_cmderror) as cma, \\\n            assert_raises(OutOfSpaceError) as cme:\n        ar.get(\"file\")\n    exc = cme.exception\n    eq_(exc.sizemore_msg, '905.6 MB')\n    assert_re_in(\".*annex (find|get). needs 905.6 MB more\", str(exc))\n\n\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_AnnexRepo_get_remote_na(path):\n    ar = AnnexRepo(path)\n\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', options=[\"--from=NotExistingRemote\"])\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n    # and similar one whenever invoking with remote parameter\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', remote=\"NotExistingRemote\")\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n\n# 1 is enough to test file_has_content\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_file_has_content(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n    testfiles = [\"test-annex.dat\", \"test.dat\"]\n\n    eq_(ar.file_has_content(testfiles), [False, False])\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.file_has_content(testfiles, batch=batch), [True, False])\n    eq_(ar.file_has_content(testfiles[:1], batch=batch), [True])\n\n    eq_(ar.file_has_content(testfiles + [\"bogus.txt\"], batch=batch),\n        [True, False, False])\n\n    assert_false(ar.file_has_content(\"bogus.txt\", batch=batch))\n    ok_(ar.file_has_content(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n# 1 is enough to test\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_is_under_annex(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n\n    with open(opj(annex_path, 'not-committed.txt'), 'w') as f:\n        f.write(\"aaa\")\n\n    testfiles = [\"test-annex.dat\", \"not-committed.txt\", \"INFO.txt\"]\n    # wouldn't change\n    target_value = [True, False, False]\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n    eq_(ar.is_under_annex(testfiles[:1], batch=batch), target_value[:1])\n    eq_(ar.is_under_annex(testfiles[1:], batch=batch), target_value[1:])\n\n    eq_(ar.is_under_annex(testfiles + [\"bogus.txt\"], batch=batch),\n                 target_value + [False])\n\n    assert_false(ar.is_under_annex(\"bogus.txt\", batch=batch))\n    ok_(ar.is_under_annex(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_web_remote(sitepath, siteurl, dst):\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = '%s_about.txt' % url_file_prefix\n    testfile2 = '%s_about2.txt' % url_file_prefix\n    testfile3 = opj('d', 'sub.txt')\n\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl])\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # output='full'\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull), set(l))  # the same entries\n    non_web_remote = l[1 - l.index(ar.WEB_UUID)]\n    assert_in('urls', lfull[non_web_remote])\n    eq_(lfull[non_web_remote]['urls'], [])\n    assert_not_in('uuid', lfull[ar.WEB_UUID])  # no uuid in the records\n    eq_(lfull[ar.WEB_UUID]['urls'], [testurl])\n\n    # --all and --key are incompatible\n    assert_raises(CommandError, ar.whereis, [], options='--all', output='full', key=True)\n\n    # output='descriptions'\n    ldesc = ar.whereis(testfile, output='descriptions')\n    eq_(set(ldesc), set([v['description'] for v in lfull.values()]))\n\n    # info w/ and w/o fast mode\n    for fast in [True, False]:\n        info = ar.info(testfile, fast=fast)\n        eq_(info['size'], 14)\n        assert(info['key'])  # that it is there\n        info_batched = ar.info(testfile, batch=True, fast=fast)\n        eq_(info, info_batched)\n        # while at it ;)\n        with swallow_outputs() as cmo:\n            eq_(ar.info('nonexistent', batch=False), None)\n            eq_(ar.info('nonexistent-batch', batch=True), None)\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n            ar.precommit()  # to stop all the batched processes for swallow_outputs\n\n    # annex repo info\n    repo_info = ar.repo_info(fast=False)\n    eq_(repo_info['local annex size'], 14)\n    eq_(repo_info['backend usage'], {'SHA256E': 1})\n    # annex repo info in fast mode\n    repo_info_fast = ar.repo_info(fast=True)\n    # doesn't give much testable info, so just comparing a subset for match with repo_info info\n    eq_(repo_info_fast['semitrusted repositories'], repo_info['semitrusted repositories'])\n    #import pprint; pprint.pprint(repo_info)\n\n    # remove the remote\n    ar.rm_url(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_not_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n\n    # now only 1 copy; drop should fail\n    res = ar.drop(testfile)\n    eq_(res['command'], 'drop')\n    eq_(res['success'], False)\n    assert_in('adjust numcopies', res['note'])\n\n    # read the url using different method\n    ar.add_url_to_file(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # 2 known copies now; drop should succeed\n    ar.drop(testfile)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n    assert_false(ar.file_has_content(testfile))\n    lfull = ar.whereis(testfile, output='full')\n    assert_not_in(non_web_remote, lfull) # not present -- so not even listed\n\n    # multiple files/urls\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl2])\n\n    # TODO: if we ask for whereis on all files, we should get for all files\n    lall = ar.whereis('.')\n    eq_(len(lall), 2)\n    for e in lall:\n        assert(isinstance(e, list))\n    # but we don't know which one for which file. need a 'full' one for that\n    lall_full = ar.whereis('.', output='full')\n    ok_(ar.file_has_content(testfile2))\n    ok_(lall_full[testfile2][non_web_remote]['here'])\n    eq_(set(lall_full), {testfile, testfile2})\n\n    # add a bogus 2nd url to testfile\n\n    someurl = \"http://example.com/someurl\"\n    ar.add_url_to_file(testfile, someurl, options=['--relaxed'])\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull[ar.WEB_UUID]['urls']), {testurl, someurl})\n\n    # and now test with a file in subdirectory\n    subdir = opj(dst, 'd')\n    os.mkdir(subdir)\n    with swallow_outputs() as cmo:\n        ar.add_url_to_file(testfile3, url=testurl3)\n    ok_file_has_content(opj(dst, testfile3), 'more stuff')\n    eq_(set(ar.whereis(testfile3)), {ar.WEB_UUID, non_web_remote})\n    eq_(set(ar.whereis(testfile3, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n\n    # and if we ask for both files\n    info2 = ar.info([testfile, testfile3])\n    eq_(set(info2), {testfile, testfile3})\n    eq_(info2[testfile3]['size'], 10)\n\n    full = ar.whereis([], options='--all', output='full')\n    eq_(len(full.keys()), 3)  # we asked for all files -- got 3 keys\n    assert_in(ar.WEB_UUID, full['SHA256E-s10--a978713ea759207f7a6f9ebc9eaebd1b40a69ae408410ddf544463f6d33a30e1.txt'])\n\n    # which would work even if we cd to that subdir, but then we should use explicit curdir\n    with chpwd(subdir):\n        cur_subfile = opj(curdir, 'sub.txt')\n        eq_(set(ar.whereis(cur_subfile)), {ar.WEB_UUID, non_web_remote})\n        eq_(set(ar.whereis(cur_subfile, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n        testfiles = [cur_subfile, opj(pardir, testfile)]\n        info2_ = ar.info(testfiles)\n        # Should maintain original relative file names\n        eq_(set(info2_), set(testfiles))\n        eq_(info2_[cur_subfile]['size'], 10)\n\n\n@with_tree(tree={\"a.txt\": \"a\",\n                 \"b\": \"b\",\n                 OBSCURE_FILENAME: \"c\",\n                 \"subdir\": {\"d\": \"d\", \"e\": \"e\"}})\ndef test_find_batch_equivalence(path):\n    ar = AnnexRepo(path)\n    files = [\"a.txt\", \"b\", OBSCURE_FILENAME]\n    ar.add(files + [\"subdir\"])\n    ar.commit(\"add files\")\n    query = [\"not-there\"] + files\n    expected = {f: f for f in files}\n    expected.update({\"not-there\": \"\"})\n    eq_(expected, ar.find(query, batch=True))\n    eq_(expected, ar.find(query))\n    # If we give a subdirectory, we split that output.\n    eq_(set(ar.find([\"subdir\"])[\"subdir\"]), {\"subdir/d\", \"subdir/e\"})\n    eq_(ar.find([\"subdir\"]), ar.find([\"subdir\"], batch=True))\n\n\n@with_tempfile(mkdir=True)\ndef test_repo_info(path):\n    repo = AnnexRepo(path)\n    info = repo.repo_info()  # works in empty repo without crashing\n    eq_(info['local annex size'], 0)\n    eq_(info['size of annexed files in working tree'], 0)\n\n    def get_custom(custom={}):\n        \"\"\"Need a helper since repo_info modifies in place so we should generate\n        new each time\n        \"\"\"\n        custom_json = {\n            'available local disk space': 'unknown',\n            'size of annexed files in working tree': \"0\",\n            'success': True,\n            'command': 'info',\n        }\n        if custom:\n            custom_json.update(custom)\n        return [custom_json]\n\n    with patch.object(\n            repo, '_run_annex_command_json',\n            return_value=get_custom()):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], None)\n\n    with patch.object(\n        repo, '_run_annex_command_json',\n        return_value=get_custom({\n            \"available local disk space\": \"19193986496 (+100000 reserved)\"})):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], 19193986496)\n\n\n@with_testrepos('.*annex.*', flavors=['local', 'network'])\n@with_tempfile\ndef test_AnnexRepo_migrating_backends(src, dst):\n    ar = AnnexRepo.clone(src, dst, backend='MD5')\n    eq_(ar.default_backends, ['MD5'])\n    # GitPython has a bug which causes .git/config being wiped out\n    # under Python3, triggered by collecting its config instance I guess\n    gc.collect()\n    ok_git_config_not_empty(ar)  # Must not blow, see https://github.com/gitpython-developers/GitPython/issues/333\n\n    filename = get_most_obscure_supported_name()\n    filename_abs = os.path.join(dst, filename)\n    f = open(filename_abs, 'w')\n    f.write(\"What to write?\")\n    f.close()\n\n    ar.add(filename, backend='MD5')\n    eq_(ar.get_file_backend(filename), 'MD5')\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n\n    # migrating will only do, if file is present\n    ok_annex_get(ar, 'test-annex.dat')\n\n    if ar.is_direct_mode():\n        # No migration in direct mode\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat')\n    else:\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n        ar.migrate_backend('test-annex.dat')\n        eq_(ar.get_file_backend('test-annex.dat'), 'MD5')\n\n        ar.migrate_backend('', backend='SHA1')\n        eq_(ar.get_file_backend(filename), 'SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n\n\ntree1args = dict(\n    tree=(\n        ('firstfile', 'whatever'),\n        ('secondfile', 'something else'),\n        ('remotefile', 'pretends to be remote'),\n        ('faraway', 'incredibly remote')),\n)\n\n# keys for files if above tree is generated and added to annex with MD5E backend\ntree1_md5e_keys = {\n    'firstfile': 'MD5E-s8--008c5926ca861023c1d2a36653fd88e2',\n    'faraway': 'MD5E-s17--5b849ed02f914d3bbb5038fe4e3fead9',\n    'secondfile': 'MD5E-s14--6c7ba9c5a141421e1c03cb9807c97c74',\n    'remotefile': 'MD5E-s21--bf7654b3de20d5926d407ea7d913deb0'\n}\n\n\n@with_tree(**tree1args)\ndef __test_get_md5s(path):\n    # was used just to generate above dict\n    annex = AnnexRepo(path, init=True, backend='MD5E')\n    files = [basename(f) for f in find_files('.*', path)]\n    annex.add(files)\n    annex.commit()\n    print({f: annex.get_file_key(f) for f in files})\n\n\n@with_batch_direct\n@with_tree(**tree1args)\ndef test_dropkey(batch, direct, path):\n    kw = {'batch': batch}\n    annex = AnnexRepo(path, init=True, backend='MD5E', direct=direct)\n    files = list(tree1_md5e_keys)\n    annex.add(files)\n    annex.commit()\n    # drop one key\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # drop multiple\n    annex.drop_key([tree1_md5e_keys[f] for f in files[1:3]], **kw)\n    # drop already dropped -- should work as well atm\n    # https://git-annex.branchable.com/bugs/dropkey_--batch_--json_--force_is_always_succesfull\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # and a mix with already dropped or not\n    annex.drop_key(list(tree1_md5e_keys.values()), **kw)\n\n\n@with_tree(**tree1args)\n@serve_path_via_http()\ndef test_AnnexRepo_backend_option(path, url):\n    ar = AnnexRepo(path, backend='MD5')\n\n    # backend recorded in .gitattributes\n    eq_(ar.get_gitattributes('.')['.']['annex.backend'], 'MD5')\n\n    ar.add('firstfile', backend='SHA1')\n    ar.add('secondfile')\n    eq_(ar.get_file_backend('firstfile'), 'SHA1')\n    eq_(ar.get_file_backend('secondfile'), 'MD5')\n\n    with swallow_outputs() as cmo:\n        # must be added under different name since annex 20160114\n        ar.add_url_to_file('remotefile2', url + 'remotefile', backend='SHA1')\n    eq_(ar.get_file_backend('remotefile2'), 'SHA1')\n\n    with swallow_outputs() as cmo:\n        ar.add_urls([url + 'faraway'], backend='SHA1')\n    # TODO: what's the annex-generated name of this?\n    # For now, workaround:\n    ok_(ar.get_file_backend(f) == 'SHA1'\n        for f in ar.get_indexed_files() if 'faraway' in f)\n\n\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_backend(src, dst):\n    #init local test-annex before cloning:\n    AnnexRepo(src)\n\n    ar = AnnexRepo.clone(src, dst)\n\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n    if not ar.is_direct_mode():\n        # no migration in direct mode\n        ok_annex_get(ar, 'test-annex.dat', network=False)\n        ar.migrate_backend('test-annex.dat', backend='SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n    else:\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat', backend='SHA1')\n\n\n@with_tempfile\ndef test_AnnexRepo_always_commit(path):\n\n    repo = AnnexRepo(path)\n    runner = Runner(cwd=path)\n    file1 = get_most_obscure_supported_name() + \"_1\"\n    file2 = get_most_obscure_supported_name() + \"_2\"\n    with open(opj(path, file1), 'w') as f:\n        f.write(\"First file.\")\n    with open(opj(path, file2), 'w') as f:\n        f.write(\"Second file.\")\n\n    # always_commit == True is expected to be default\n    repo.add(file1)\n\n    # Now git-annex log should show the addition:\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 1)\n    assert_in(file1, out_list[0])\n    # check git log of git-annex branch:\n    # expected: initial creation, update (by annex add) and another\n    # update (by annex log)\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = False\n    repo.add(file2)\n\n    # No additional git commit:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = True\n\n    # Still one commit only in git-annex log,\n    # but 'git annex log' was called when always_commit was true again,\n    # so it should commit the addition at the end. Calling it again should then\n    # show two commits.\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(\"recording state in git\", out_list[1])\n\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(file2, out_list[1])\n\n    # Now git knows as well:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 4)\n\n\n@with_testrepos('basic_annex', flavors=['local'])\n@with_tempfile\ndef test_AnnexRepo_on_uninited_annex(origin, path):\n    # \"Manually\" clone to avoid initialization:\n    from datalad.cmd import Runner\n    runner = Runner()\n    _ = runner([\"git\", \"clone\", origin, path], expect_stderr=True)\n\n    assert_false(exists(opj(path, '.git', 'annex'))) # must not be there for this test to be valid\n    annex = AnnexRepo(path, create=False, init=False)  # so we can initialize without\n    # and still can get our things\n    assert_false(annex.file_has_content('test-annex.dat'))\n    with swallow_outputs():\n        annex.get('test-annex.dat')\n        ok_(annex.file_has_content('test-annex.dat'))\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_commit(path):\n\n    ds = AnnexRepo(path, create=True)\n    filename = opj(path, get_most_obscure_supported_name())\n    with open(filename, 'w') as f:\n        f.write(\"File to add to git\")\n    ds.add(filename, git=True)\n\n    assert_raises(AssertionError, ok_clean_git, path, annex=True)\n\n    ds.commit(\"test _commit\")\n    ok_clean_git(path, annex=True)\n\n    # nothing to commit doesn't raise by default:\n    ds.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, ds.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(opj(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"not-existing\")\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_annex(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    filename_abs = opj(repo.path, filename)\n    with open(filename_abs, \"w\") as f:\n        f.write(\"some\")\n\n    out_json = repo.add(filename)\n    # file is known to annex:\n    if not repo.is_direct_mode():\n        assert_true(os.path.islink(filename_abs),\n                    \"Annexed file is not a link.\")\n    else:\n        assert_false(os.path.islink(filename_abs),\n                     \"Annexed file is link in direct mode.\")\n    assert_in('key', out_json)\n    key = repo.get_file_key(filename)\n    assert_false(key == '')\n    assert_equal(key, out_json['key'])\n    ok_(repo.file_has_content(filename))\n\n    # uncommitted:\n    # but not in direct mode branch\n    if repo.is_direct_mode():\n        ok_(not repo.is_dirty(submodules=False))\n    else:\n        ok_(repo.is_dirty(submodules=False))\n\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename)\n    repo.commit(msg=\"Added another file to annex.\")\n    # known to annex:\n    ok_(repo.get_file_key(filename))\n    ok_(repo.file_has_content(filename))\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_git(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"some\")\n    repo.add(filename, git=True)\n\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n    # uncommitted:\n    ok_(repo.is_dirty(submodules=False))\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename, git=True)\n    repo.commit(msg=\"Added another file to annex.\")\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_AnnexRepo_add_unexpected_direct_mode(path):\n    # tests a special case where a submodule is in direct mode, while it's\n    # superproject is not.\n    # There is no point in this test, if direct mode was enforced in the\n    # superproject already (either by test run configuration or FS) or if the\n    # repositories are in V6 by default (where there is no direct mode)\n\n    top = AnnexRepo(path)\n\n    if top.is_direct_mode() or top.config.get(\"annex.version\") == '6':\n        raise SkipTest(\"Nothing to test for\")\n\n    top.update_submodule('subm 1', init=True)\n    sub = AnnexRepo(opj(path, 'subm 1'))\n    sub.set_direct_mode(True)\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        top.add('.')\n        cml.assert_logged(msg=\"Known bug in direct mode.\",\n                          level=\"WARNING\",\n                          regex=False)\n\n\n\n@ignore_nose_capturing_stdout\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_AnnexRepo_get(src, dst):\n\n    annex = AnnexRepo.clone(src, dst)\n    assert_is_instance(annex, AnnexRepo, \"AnnexRepo was not created.\")\n    testfile = 'test-annex.dat'\n    testfile_abs = opj(dst, testfile)\n    assert_false(annex.file_has_content(\"test-annex.dat\"))\n    with swallow_outputs():\n        annex.get(testfile)\n    ok_(annex.file_has_content(\"test-annex.dat\"))\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n    called = []\n    # for some reason yoh failed mock to properly just call original func\n    orig_run = annex._run_annex_command\n\n    def check_run(cmd, annex_options, **kwargs):\n        called.append(cmd)\n        if cmd == 'find':\n            assert_not_in('-J5', annex_options)\n        elif cmd == 'get':\n            assert_in('-J5', annex_options)\n        else:\n            raise AssertionError(\n                \"no other commands so far should be ran. Got %s, %s\" %\n                (cmd, annex_options)\n            )\n        return orig_run(cmd, annex_options=annex_options, **kwargs)\n\n    annex.drop(testfile)\n    with patch.object(AnnexRepo, '_run_annex_command',\n                      side_effect=check_run, auto_spec=True), \\\n            swallow_outputs():\n        annex.get(testfile, jobs=5)\n    eq_(called, ['find', 'get'])\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n\n# TODO:\n#def init_remote(self, name, options):\n#def enable_remote(self, name):\n\n@with_testrepos('basic_annex$', flavors=['clone'])\n@with_tempfile\ndef _test_AnnexRepo_get_contentlocation(batch, path, work_dir_outside):\n    annex = AnnexRepo(path, create=False, init=False)\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n    # TODO: see if we can avoid this or specify custom exception\n    eq_(annex.get_contentlocation(key, batch=batch), '')\n\n    with swallow_outputs() as cmo:\n        annex.get(fname)\n    key_location = annex.get_contentlocation(key, batch=batch)\n    assert(key_location)\n    # they both should point to the same location eventually\n    eq_(os.path.realpath(opj(annex.path, fname)),\n        os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a subdir of the annex:\n    with chpwd(opj(annex.path, 'subdir'), mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a dir outside of the annex:\n    with chpwd(work_dir_outside, mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n\ndef test_AnnexRepo_get_contentlocation():\n    for batch in (False, True):\n        yield _test_AnnexRepo_get_contentlocation, batch\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('about2_.txt', 'more abouts_'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_to_file_batched(sitepath, siteurl, dst):\n\n    if os.environ.get('DATALAD_FAKE__DATES'):\n        raise SkipTest(\n            \"Faked dates are enabled; skipping batched addurl tests\")\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl2_ = urljoin(siteurl, 'about2_.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = 'about.txt'\n    testfile2 = 'about2.txt'\n    testfile2_ = 'about2_.txt'\n    testfile3 = opj('d', 'sub.txt')\n\n    # add to an existing but not committed file\n    # TODO: __call__ of the BatchedAnnex must be checked to be called\n    copyfile(opj(sitepath, 'about.txt'), opj(dst, testfile))\n    # must crash sensibly since file exists, we shouldn't addurl to non-annexed files\n    with assert_raises(AnnexBatchCommandError):\n        ar.add_url_to_file(testfile, testurl, batch=True)\n\n    # Remove it and re-add\n    unlink(opj(dst, testfile))\n    ar.add_url_to_file(testfile, testurl, batch=True)\n\n    info = ar.info(testfile)\n    eq_(info['size'], 14)\n    assert(info['key'])\n    # not even added to index yet since we this repo is with default batch_size\n    # but: in direct mode it is added!\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n\n    # TODO: none of the below should re-initiate the batch process\n\n    # add to an existing and staged annex file\n    copyfile(opj(sitepath, 'about2.txt'), opj(dst, testfile2))\n    ar.add(testfile2)\n    ar.add_url_to_file(testfile2, testurl2, batch=True)\n    assert(ar.info(testfile2))\n    # not committed yet\n    # assert_in(ar.WEB_UUID, ar.whereis(testfile2))\n\n    # add to an existing and committed annex file\n    copyfile(opj(sitepath, 'about2_.txt'), opj(dst, testfile2_))\n    ar.add(testfile2_)\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n    ar.commit(\"added about2_.txt and there was about2.txt lingering around\")\n    # commit causes closing all batched annexes, so testfile gets committed\n    assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    assert(not ar.dirty)\n    ar.add_url_to_file(testfile2_, testurl2_, batch=True)\n    assert(ar.info(testfile2_))\n    assert_in(ar.WEB_UUID, ar.whereis(testfile2_))\n\n    # add into a new file\n    # filename = 'newfile.dat'\n    filename = get_most_obscure_supported_name()\n\n    # Note: The following line was necessary, since the test setup just\n    # doesn't work with singletons\n    # TODO: Singleton mechanic needs a general solution for this\n    AnnexRepo._unique_instances.clear()\n    ar2 = AnnexRepo(dst, batch_size=1)\n\n    with swallow_outputs():\n        eq_(len(ar2._batched), 0)\n        ar2.add_url_to_file(filename, testurl, batch=True)\n        eq_(len(ar2._batched), 1)  # we added one more with batch_size=1\n        ar2.precommit()  # to possibly stop batch process occupying the stdout\n    ar2.commit(\"added new file\")  # would do nothing ATM, but also doesn't fail\n    assert_in(filename, ar2.get_files())\n    assert_in(ar.WEB_UUID, ar2.whereis(filename))\n\n    if not ar.is_direct_mode():\n        # in direct mode there's nothing to commit\n        ar.commit(\"actually committing new files\")\n    assert_in(filename, ar.get_files())\n    assert_in(ar.WEB_UUID, ar.whereis(filename))\n    # this poor bugger still wasn't added since we used default batch_size=0 on him\n\n    # and closing the pipes now shoudn't anyhow affect things\n    eq_(len(ar._batched), 1)\n    ar._batched.close()\n    eq_(len(ar._batched), 1)  # doesn't remove them, just closes\n    assert(not ar.dirty)\n\n    ar._batched.clear()\n    eq_(len(ar._batched), 0)  # .clear also removes\n\n    raise SkipTest(\"TODO: more, e.g. add with a custom backend\")\n    # TODO: also with different modes (relaxed, fast)\n    # TODO: verify that file is added with that backend and that we got a new batched process\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\n@serve_path_via_http()\n@with_tree(tree={\"bar\": \"bar content\"})\ndef test_annexrepo_fake_dates_disables_batched(sitepath, siteurl, dst):\n    ar = AnnexRepo(dst, create=True, fake_dates=True)\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.add_url_to_file(\"foo-dst\", urljoin(siteurl, \"foo\"), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching addurl call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n    ar.add(\"bar\")\n    ar.commit(\"add bar\")\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.drop_key(ar.get_file_key([\"bar\"]), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching drop_key call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_backends(path):\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, None)\n\n    rmtree(path)\n\n    repo = AnnexRepo(path, backend='MD5E')\n    eq_(repo.default_backends, ['MD5E'])\n\n    # persists\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, ['MD5E'])\n\n\n@skip_ssh\n@with_tempfile\n@with_testrepos('basic_annex', flavors=['local'])\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_annex_ssh(repo_path, remote_1_path, remote_2_path):\n    from datalad import ssh_manager\n    # create remotes:\n    rm1 = AnnexRepo(remote_1_path, create=False)\n    rm2 = AnnexRepo(remote_2_path, create=False)\n\n    # check whether we are the first to use these sockets:\n    socket_1 = opj(ssh_manager.socket_dir, get_connection_hash('datalad-test'))\n    socket_2 = opj(ssh_manager.socket_dir, get_connection_hash('localhost'))\n    datalad_test_was_open = exists(socket_1)\n    localhost_was_open = exists(socket_2)\n\n    # repo to test:AnnexRepo(repo_path)\n    # At first, directly use git to add the remote, which should be recognized\n    # by AnnexRepo's constructor\n    gr = GitRepo(repo_path, create=True)\n    AnnexRepo(repo_path)\n    gr.add_remote(\"ssh-remote-1\", \"ssh://datalad-test\" + remote_1_path)\n\n    # Now, make it an annex:\n    ar = AnnexRepo(repo_path, create=False)\n\n    # connection to 'datalad-test' should be known to ssh manager:\n    assert_in(socket_1, ssh_manager._connections)\n    # but socket was not touched:\n    if datalad_test_was_open:\n        ok_(exists(socket_1))\n    else:\n        ok_(not exists(socket_1))\n\n    from datalad import lgr\n    # remote interaction causes socket to be created:\n    try:\n        # Note: For some reason, it hangs if log_stdout/err True\n        # TODO: Figure out what's going on\n        #  yoh: I think it is because of what is \"TODOed\" within cmd.py --\n        #       trying to log/obtain both through PIPE could lead to lock\n        #       downs.\n        # here we use our swallow_logs to overcome a problem of running under\n        # nosetests without -s, when nose then tries to swallow stdout by\n        # mocking it with StringIO, which is not fully compatible with Popen\n        # which needs its .fileno()\n        with swallow_outputs():\n            ar._run_annex_command('sync',\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_1))\n\n    # add another remote:\n    ar.add_remote('ssh-remote-2', \"ssh://localhost\" + remote_2_path)\n\n    # now, this connection to localhost was requested:\n    assert_in(socket_2, ssh_manager._connections)\n    # but socket was not touched:\n    if localhost_was_open:\n        # FIXME: occasionally(?) fails in V6:\n        if not ar.config.getint(\"annex\", \"version\") == 6:\n            ok_(exists(socket_2))\n    else:\n        ok_(not exists(socket_2))\n\n    # sync with the new remote:\n    try:\n        with swallow_outputs():\n            ar._run_annex_command('sync', annex_options=['ssh-remote-2'],\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_2))\n    ssh_manager.close(ctrl_path=[socket_1, socket_2])\n\n@with_testrepos('basic_annex', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_remove(path1, path2):\n    ar1 = AnnexRepo(path1, create=False)\n    ar2 = AnnexRepo.clone(path1, path2, create=True, direct=True)\n\n    for repo in (ar1, ar2):\n        file_list = repo.get_annexed_files()\n        assert len(file_list) >= 1\n        # remove a single file\n        out = repo.remove(file_list[0])\n        assert_not_in(file_list[0], repo.get_annexed_files())\n        eq_(out[0], file_list[0])\n\n        with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n            f.write(\"whatever\")\n\n        # add it\n        repo.add(\"rm-test.dat\")\n\n        # remove without '--force' should fail, due to staged changes:\n        if repo.is_direct_mode():\n            assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n        else:\n            assert_raises(GitCommandError, repo.remove, \"rm-test.dat\")\n        assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n        # now force:\n        out = repo.remove(\"rm-test.dat\", force=True)\n        assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n        eq_(out[0], \"rm-test.dat\")\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_repo_version(path1, path2, path3):\n    annex = AnnexRepo(path1, create=True, version=6)\n    ok_clean_git(path1, annex=True)\n    version = annex.repo.config_reader().get_value('annex', 'version')\n    eq_(version, 6)\n\n    # default from config item (via env var):\n    with patch.dict('os.environ', {'DATALAD_REPO_VERSION': '6'}):\n        annex = AnnexRepo(path2, create=True)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 6)\n\n        # parameter `version` still has priority over default config:\n        annex = AnnexRepo(path3, create=True, version=5)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 5)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_copy_to(origin, clone):\n    repo = AnnexRepo(origin, create=False)\n    remote = AnnexRepo.clone(origin, clone, create=True)\n    repo.add_remote(\"target\", clone)\n\n    assert_raises(IOError, repo.copy_to, \"doesnt_exist.dat\", \"target\")\n    assert_raises(FileInGitError, repo.copy_to, \"INFO.txt\", \"target\")\n    assert_raises(ValueError, repo.copy_to, \"test-annex.dat\", \"invalid_target\")\n\n    # test-annex.dat has no content to copy yet:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [])\n\n    repo.get(\"test-annex.dat\")\n    # now it has:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [\"test-annex.dat\"])\n    # and will not be copied again since it was already copied\n    eq_(repo.copy_to([\"INFO.txt\", \"test-annex.dat\"], \"target\"), [])\n\n    # Test that if we pass a list of items and annex processes them nicely,\n    # we would obtain a list back. To not stress our tests even more -- let's mock\n    def ok_copy(command, **kwargs):\n        # Check that we do pass to annex call only the list of files which we\n        #  asked to be copied\n        assert_in('copied1', kwargs['annex_options'])\n        assert_in('copied2', kwargs['annex_options'])\n        assert_in('existed', kwargs['annex_options'])\n        return \"\"\"\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied1\"}\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"copied2\"}\n{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey3\", \"file\":\"existed\"}\n\"\"\", \"\"\n    with patch.object(repo, '_run_annex_command', ok_copy):\n        eq_(repo.copy_to([\"copied2\", \"copied1\", \"existed\"], \"target\"),\n            [\"copied1\", \"copied2\"])\n\n    # now let's test that we are correctly raising the exception in case if\n    # git-annex execution fails\n    orig_run = repo._run_annex_command\n\n    # Kinda a bit off the reality since no nonex* would not be returned/handled\n    # by _get_expected_files, so in real life -- wouldn't get report about Incomplete!?\n    def fail_to_copy(command, **kwargs):\n        if command == 'copy':\n            # That is not how annex behaves\n            # http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            # for non-existing files output goes into stderr\n            raise CommandError(\n                \"Failed to run ...\",\n                stdout=\n                    '{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied\"}\\n'\n                    '{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"existed\"}\\n',\n                stderr=\n                    'git-annex: nonex1 not found\\n'\n                    'git-annex: nonex2 not found\\n'\n            )\n        else:\n            return orig_run(command, **kwargs)\n\n    def fail_to_copy_get_expected(files, expr):\n        assert files == [\"copied\", \"existed\", \"nonex1\", \"nonex2\"]\n        return {'akey1': 10}, [\"copied\"]\n\n    with patch.object(repo, '_run_annex_command', fail_to_copy), \\\n            patch.object(repo, '_get_expected_files', fail_to_copy_get_expected):\n        with assert_raises(IncompleteResultsError) as cme:\n            repo.copy_to([\"copied\", \"existed\", \"nonex1\", \"nonex2\"], \"target\")\n    eq_(cme.exception.results, [\"copied\"])\n    eq_(cme.exception.failed, ['nonex1', 'nonex2'])\n\n\n\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_annex_drop(src, dst):\n    ar = AnnexRepo.clone(src, dst)\n    testfile = 'test-annex.dat'\n    assert_false(ar.file_has_content(testfile))\n    ar.get(testfile)\n    ok_(ar.file_has_content(testfile))\n\n    # drop file by name:\n    result = ar.drop([testfile])\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['file'], testfile)\n\n    ar.get(testfile)\n\n    # drop file by key:\n    testkey = ar.get_file_key(testfile)\n    result = ar.drop([testkey], key=True)\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['key'], testkey)\n\n    # insufficient arguments:\n    assert_raises(TypeError, ar.drop)\n    assert_raises(InsufficientArgumentsError, ar.drop, [], options=[\"--jobs=5\"])\n    assert_raises(InsufficientArgumentsError, ar.drop, [])\n\n    # too much arguments:\n    assert_raises(CommandError, ar.drop, ['.'], options=['--all'])\n\n\n@with_tree({\"a.txt\": \"a\", \"b.txt\": \"b\", \"c.py\": \"c\", \"d\": \"d\"})\ndef test_annex_get_annexed_files(path):\n    repo = AnnexRepo(path)\n    repo.add(\".\")\n    repo.commit()\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n\n    repo.drop(\"a.txt\", options=[\"--force\"])\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        {\"b.txt\", \"c.py\", \"d\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\"])),\n        {\"a.txt\", \"b.txt\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True,\n                                   patterns=[\"*.txt\"])),\n        {\"b.txt\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\", \"*.py\"])),\n        {\"a.txt\", \"b.txt\", \"c.py\"})\n\n    eq_(set(repo.get_annexed_files()),\n        set(repo.get_annexed_files(patterns=[\"*\"])))\n\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        set(repo.get_annexed_files(with_content_only=True, patterns=[\"*\"])))\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_annex_remove(path):\n    repo = AnnexRepo(path, create=False)\n\n    file_list = repo.get_annexed_files()\n    assert len(file_list) >= 1\n    # remove a single file\n    out = repo.remove(file_list[0])\n    assert_not_in(file_list[0], repo.get_annexed_files())\n    eq_(out[0], file_list[0])\n\n    with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    # add it\n    repo.add(\"rm-test.dat\")\n\n    # remove without '--force' should fail, due to staged changes:\n    assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n    assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n    # now force:\n    out = repo.remove(\"rm-test.dat\", force=True)\n    assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n    eq_(out[0], \"rm-test.dat\")\n\n\n@with_batch_direct\n@with_testrepos('basic_annex', flavors=['clone'], count=1)\ndef test_is_available(batch, direct, p):\n    annex = AnnexRepo(p)\n\n    # bkw = {'batch': batch}\n    if batch:\n        is_available = partial(annex.is_available, batch=batch)\n    else:\n        is_available = annex.is_available\n\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n\n    # explicit is to verify data type etc\n    assert is_available(key, key=True) is True\n    assert is_available(fname) is True\n\n    # known remote but doesn't have it\n    assert is_available(fname, remote='origin') is False\n    # it is on the 'web'\n    assert is_available(fname, remote='web') is True\n    # not effective somehow :-/  may be the process already running or smth\n    # with swallow_logs(), swallow_outputs():  # it will complain!\n    assert is_available(fname, remote='unknown') is False\n    assert_false(is_available(\"boguskey\", key=True))\n\n    # remove url\n    urls = annex.get_urls(fname) #, **bkw)\n    assert(len(urls) == 1)\n    annex.rm_url(fname, urls[0])\n\n    assert is_available(key, key=True) is False\n    assert is_available(fname) is False\n    assert is_available(fname, remote='web') is False\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_add_no_dotfiles(path):\n    ar = AnnexRepo(path, create=True)\n    print(ar.path)\n    assert_true(os.path.exists(ar.path))\n    assert_false(ar.dirty)\n    os.makedirs(opj(ar.path, '.datalad'))\n    # we don't care about empty directories\n    assert_false(ar.dirty)\n    with open(opj(ar.path, '.datalad', 'somefile'), 'w') as f:\n        f.write('some content')\n    # make sure the repo is considered dirty now\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # no file is being added, as dotfiles/directories are ignored by default\n    ar.add('.', git=False)\n    # double check, still dirty\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # now add to git, and it should work\n    ar.add('.', git=True)\n    # all in index\n    assert_true(ar.dirty)\n    # TODO: has been more specific:\n    # assert_false(ar.repo.is_dirty(\n    #     index=False, working_tree=True, untracked_files=True, submodules=True))\n    ar.commit(msg=\"some\")\n    # all committed\n    assert_false(ar.dirty)\n    # not known to annex\n    assert_false(ar.is_under_annex(opj(ar.path, '.datalad', 'somefile')))\n\n\n@with_tempfile\ndef test_annex_version_handling(path):\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n         patch.object(AnnexRepo, '_check_git_annex_version',\n                      auto_spec=True,\n                      side_effect=AnnexRepo._check_git_annex_version) \\\n            as cmpc, \\\n         patch.object(external_versions, '_versions',\n                      {'cmd:annex': AnnexRepo.GIT_ANNEX_MIN_VERSION}):\n            eq_(AnnexRepo.git_annex_version, None)\n            ar1 = AnnexRepo(path, create=True)\n            assert(ar1)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n            # 2nd time must not be called\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            ar2 = AnnexRepo(path)\n            assert(ar2)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n            patch.object(AnnexRepo, '_check_git_annex_version',\n                         auto_spec=True,\n                         side_effect=AnnexRepo._check_git_annex_version):\n        # no git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': None}):\n            eq_(AnnexRepo.git_annex_version, None)\n            with assert_raises(MissingExternalDependency) as cme:\n                try:\n                    # Note: Remove to cause creation of a new instance\n                    rmtree(path)\n                except OSError:\n                    pass\n                AnnexRepo(path)\n            if linux_distribution_name == 'debian':\n                assert_in(\"http://neuro.debian.net\", str(cme.exception))\n            eq_(AnnexRepo.git_annex_version, None)\n\n        # outdated git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': '6.20160505'}):\n            eq_(AnnexRepo.git_annex_version, None)\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n            # and we don't assign it\n            eq_(AnnexRepo.git_annex_version, None)\n            # so we could still fail\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n\n\ndef test_ProcessAnnexProgressIndicators():\n    irrelevant_lines = (\n        'abra',\n        '{\"some_json\": \"sure thing\"}'\n    )\n    # regular lines, without completion for known downloads\n    success_lines = (\n        '{\"command\":\"get\",\"note\":\"\",\"success\":true,\"key\":\"key1\",\"file\":\"file1\"}',\n        '{\"command\":\"comm\",\"note\":\"\",\"success\":true,\"key\":\"backend-s10--key2\"}',\n    )\n    progress_lines = (\n        '{\"byte-progress\":10,\"action\":{\"command\":\"get\",\"note\":\"from web...\",'\n            '\"key\":\"key1\",\"file\":\"file1\"},\"percent-progress\":\"10%\"}',\n    )\n\n    # without providing expected entries\n    proc = ProcessAnnexProgressIndicators()\n    # when without any target downloads, there is no total_pbar\n    assert_is(proc.total_pbar, None)\n    # for regular lines -- should just return them without side-effects\n    for l in irrelevant_lines + success_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    eq_(proc(progress_lines[0]), None)\n    eq_(len(proc.pbars), 1)\n    # but when we finish download -- should get cleared\n    eq_(proc(success_lines[0]), success_lines[0])\n    eq_(proc.pbars, {})\n    # and no side-effect of any kind in finish\n    eq_(proc.finish(), None)\n\n    proc = ProcessAnnexProgressIndicators(expected={'key1': 100, 'key2': None})\n    # when without any target downloads, there is no total_pbar\n    assert(proc.total_pbar is not None)\n    eq_(proc.total_pbar.total, 100)  # as much as it knows at this point\n    eq_(proc.total_pbar.current, 0)\n    # for regular lines -- should still just return them without side-effects\n    for l in irrelevant_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    # it doesn't swallow everything -- so there will be side-effects in output\n    with swallow_outputs() as cmo:\n        eq_(proc(progress_lines[0]), None)\n        eq_(len(proc.pbars), 1)\n        # but when we finish download -- should get cleared\n        eq_(proc(success_lines[0]), success_lines[0])\n        eq_(proc.pbars, {})\n        out = cmo.out\n\n    from datalad.ui import ui\n    from datalad.ui.dialog import SilentConsoleLog\n\n    assert out \\\n        if not isinstance(ui.ui, SilentConsoleLog) else not out\n    assert proc.total_pbar is not None\n    # and no side-effect of any kind in finish\n    with swallow_outputs() as cmo:\n        eq_(proc.finish(), None)\n        eq_(proc.total_pbar, None)\n\n\n@with_tempfile\n@with_tempfile\ndef test_get_description(path1, path2):\n    annex1 = AnnexRepo(path1, create=True)\n    # some content for git-annex branch\n    create_tree(path1, {'1.dat': 'content'})\n    annex1.add('1.dat', git=False)\n    annex1.commit(\"msg\")\n    annex1_description = annex1.get_description()\n    assert_not_equal(annex1_description, path1)\n\n    annex2 = AnnexRepo(path2, create=True, description='custom 2')\n    eq_(annex2.get_description(), 'custom 2')\n    # not yet known\n    eq_(annex2.get_description(uuid=annex1.uuid), None)\n\n    annex2.add_remote('annex1', path1)\n    annex2.fetch('annex1')\n    # it will match the remote name\n    eq_(annex2.get_description(uuid=annex1.uuid),\n        annex1_description + ' [annex1]')\n    # add a little probe file to make sure it stays untracked\n    create_tree(path1, {'probe': 'probe'})\n    assert_not_in('probe', annex2.get_indexed_files())\n    annex2.merge_annex('annex1')\n    assert_not_in('probe', annex2.get_indexed_files())\n    # but let's remove the remote\n    annex2.remove_remote('annex1')\n    eq_(annex2.get_description(uuid=annex1.uuid), annex1_description)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_flyweight(path1, path2):\n\n    repo1 = AnnexRepo(path1, create=True)\n    assert_is_instance(repo1, AnnexRepo)\n    # instantiate again:\n    repo2 = AnnexRepo(path1, create=False)\n    assert_is_instance(repo2, AnnexRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in an different way:\n    with chpwd(path1):\n        repo3 = AnnexRepo(relpath(path1, start=path2), create=False)\n        assert_is_instance(repo3, AnnexRepo)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # but path attribute is absolute, so they are still equal:\n    ok_(repo1 == repo3)\n\n    # Now, let's try to get a GitRepo instance from a path, we already have an\n    # AnnexRepo of\n    repo4 = GitRepo(path1)\n    assert_is_instance(repo4, GitRepo)\n    assert_not_is_instance(repo4, AnnexRepo)\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_AnnexRepo_get_toppath(repo, tempdir, repo2):\n\n    reporeal = realpath(repo)\n    eq_(AnnexRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(AnnexRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    AnnexRepo(repo2, create=True)\n    repo2real = realpath(repo2)\n    nested = opj(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(AnnexRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(AnnexRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(AnnexRepo.get_toppath(tempdir), None)\n\n\n@with_testrepos(\".*basic.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_add_submodule(source, path):\n\n    top_repo = AnnexRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n\n    ok_clean_git(top_repo, annex=True)\n    ok_clean_git(opj(path, 'sub'), annex=False)\n\n\ndef test_AnnexRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@known_failure_v6  #FIXME\ndef test_AnnexRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_dirty(path):\n\n    repo = AnnexRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # pure git operations:\n    # untracked file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt', git=True)\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    if not repo.config.getint(\"annex\", \"version\") == 6:\n        ok_(not repo.dirty)\n    # modified file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt', git=True)\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # annex operations:\n    # untracked file\n    with open(opj(path, 'file2.txt'), 'w') as f:\n        f.write('different content')\n    ok_(repo.dirty)\n    # annexed file\n    repo.add('file2.txt', git=False)\n    if not repo.is_direct_mode():\n        # in direct mode 'annex add' results in a clean repo\n        ok_(repo.dirty)\n        # commit\n        repo.commit(\"file2.txt annexed\")\n    ok_(not repo.dirty)\n\n    # TODO: unlock/modify\n\n    # TODO: submodules\n\n\ndef _test_status(ar):\n    # TODO: plain git submodule and even deeper hierarchy?\n    #       => complete recursion to work if started from within plain git;\n    #       But this is then relevant for Dataset.status() - not herein\n\n    def sync_wrapper(push=False, pull=False, commit=False):\n        # wraps common annex-sync call, since it currently fails under\n        # mysterious circumstances in V6 adjusted branch setups\n        try:\n            ar.sync(push=push, pull=pull, commit=commit)\n        except CommandError as e:\n            if \"fatal: entry 'submod' object type (blob) doesn't match mode type \" \\\n               \"(commit)\" in e.stderr:\n                # some bug in adjusted branch(?) + submodule\n                # TODO: figure out and probably report\n                # stdout:\n                # commit\n                # [adjusted/master(unlocked) ae3e9a7] git-annex in ben@tree:/tmp/datalad_temp_test_AnnexRepo_status7qKhRQ\n                #  4 files changed, 4 insertions(+), 2 deletions(-)\n                #  create mode 100644 fifth\n                #  create mode 100644 sub/third\n                # ok\n                #\n                # failed\n                # stderr:\n                # fatal: entry 'submod' object type (blob) doesn't match mode type (commit)\n                # git-annex: user error (git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"mktree\",\"--batch\",\"-z\"] exited 128)\n                # git-annex: sync: 1 failed\n\n                # But it almost works - so apperently nothing to do\n                import logging\n                lgr = logging.getLogger(\"datalad.support.tests.test-status\")\n                lgr.warning(\"DEBUG: v6 sync failure\")\n\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n\n    # untracked files:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"it's huge!\")\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"looser\")\n\n    stat['untracked'].append('first')\n    stat['untracked'].append('second')\n    eq_(stat, ar.get_status())\n\n    # add a file to git\n    ar.add('first', git=True)\n    sync_wrapper()\n    stat['untracked'].remove('first')\n    stat['added'].append('first')\n    eq_(stat, ar.get_status())\n\n    # add a file to annex\n    ar.add('second')\n    sync_wrapper()\n    stat['untracked'].remove('second')\n    if not ar.is_direct_mode():\n        # in direct mode annex-status doesn't report an added file 'added'\n        stat['added'].append('second')\n    eq_(stat, ar.get_status())\n\n    # commit to be clean again:\n    ar.commit(\"added first and second\")\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n    # create a file to be unannexed:\n    with open(opj(ar.path, 'fifth'), 'w') as f:\n        f.write(\"total disaster\")\n\n    ar.add('fifth')\n    sync_wrapper()\n    # TODO:\n    # Note: For some reason this seems to be the only place, where we actually\n    # need to call commit via annex-proxy. If called via '-c core.bare=False'\n    # and/or '--work-tree=.' the file ends up in git instead of annex.\n    # Note 2: This is only if we explicitly pass a path. Otherwise it works\n    # without annex-proxy.\n    ar.commit(msg=\"fifth to be unannexed\", files='fifth',\n              proxy=ar.is_direct_mode())\n    eq_(stat, ar.get_status())\n\n    ar.unannex('fifth')\n\n    sync_wrapper(pull=False, push=False, commit=True)\n    stat['untracked'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # modify a file in git:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"increased tremendousness\")\n    stat['modified'].append('first')\n    eq_(stat, ar.get_status())\n\n    # modify an annexed file:\n    if not ar.is_direct_mode():\n        # actually: if 'second' isn't locked, which is the case in direct mode\n        ar.unlock('second')\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].append('second')\n        eq_(stat, ar.get_status())\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"Needed to unlock first. Sad!\")\n    if not ar.is_direct_mode():\n        ar.add('second')  # => modified\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].remove('second')\n    stat['modified'].append('second')\n    sync_wrapper()\n    eq_(stat, ar.get_status())\n\n    # create something in a subdir\n    os.mkdir(opj(ar.path, 'sub'))\n    with open(opj(ar.path, 'sub', 'third'), 'w') as f:\n        f.write(\"tired of winning\")\n\n    # Note, that this is different from 'git status',\n    # which would just say 'sub/':\n    stat['untracked'].append(opj('sub', 'third'))\n    eq_(stat, ar.get_status())\n\n    # test parameters for status to restrict results:\n    # limit requested states:\n    limited_status = ar.get_status(untracked=True, deleted=False, modified=True,\n                                   added=True, type_changed=False)\n    eq_(len(limited_status), 3)\n    ok_(all([k in ('untracked', 'modified', 'added') for k in limited_status]))\n    eq_(stat['untracked'], limited_status['untracked'])\n    eq_(stat['modified'], limited_status['modified'])\n    eq_(stat['added'], limited_status['added'])\n    # limit requested files:\n    limited_status = ar.get_status(path=opj('sub', 'third'))\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    ok_(all([len(limited_status[l]) == 0 for l in ('modified', 'added',\n                                                   'deleted', 'type_changed')]))\n    # again, with a list:\n    limited_status = ar.get_status(path=[opj('sub', 'third'), 'second'])\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    eq_(limited_status['modified'], ['second'])\n    ok_(all([len(limited_status[l]) == 0 for l in ('added', 'deleted',\n                                                   'type_changed')]))\n\n    # create a subrepo:\n    sub = AnnexRepo(opj(ar.path, 'submod'), create=True)\n    # nothing changed, it's empty besides .git, which is ignored\n    eq_(stat, ar.get_status())\n\n    # file in subrepo\n    with open(opj(ar.path, 'submod', 'fourth'), 'w') as f:\n        f.write(\"this is a birth certificate\")\n    stat['untracked'].append(opj('submod', 'fourth'))\n    eq_(stat, ar.get_status())\n\n    # add to subrepo\n    sub.add('fourth')\n    sub.commit(msg=\"birther mod init'ed\")\n    stat['untracked'].remove(opj('submod', 'fourth'))\n\n    if ar.get_active_branch().endswith('(unlocked)') and \\\n       'adjusted' in ar.get_active_branch():\n        # we are running on adjusted branch => do it in submodule, too\n        sub.adjust()\n\n    # Note, that now the non-empty repo is untracked\n    stat['untracked'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the submodule\n    ar.add_submodule('submod', url=opj(curdir, 'submod'))\n\n    stat['untracked'].remove('submod/')\n    stat['added'].append('.gitmodules')\n\n    # 'submod/' might either be reported as 'added' or 'modified'.\n    # Therefore more complex assertions at this point:\n    reported_stat = ar.get_status()\n    eq_(stat['untracked'], reported_stat['untracked'])\n    eq_(stat['deleted'], reported_stat['deleted'])\n    eq_(stat['type_changed'], reported_stat['type_changed'])\n    ok_(stat['added'] == reported_stat['added'] or\n        stat['added'] + ['submod/'] == reported_stat['added'])\n    ok_(stat['modified'] == reported_stat['modified'] or\n        stat['modified'] + ['submod/'] == reported_stat['modified'])\n\n    # simpler assertion if we ignore submodules:\n    eq_(stat, ar.get_status(submodules=False))\n\n    # commit the submodule\n    # in direct mode, commit of a removed submodule fails with:\n    #  error: unable to index file submod\n    #  fatal: updating files failed\n    #\n    # - this happens, when commit is called with -c core.bare=False\n    # - it works when called via annex proxy\n    # - if we add a submodule instead of removing one, it's vice versa with\n    #   the very same error message\n\n    ar.commit(msg=\"submodule added\", files=['.gitmodules', 'submod'])\n\n    stat['added'].remove('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # add another file to submodule\n    with open(opj(ar.path, 'submod', 'not_tracked'), 'w') as f:\n        f.write(\"#LastNightInSweden\")\n    stat['modified'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the untracked file:\n    sub.add('not_tracked')\n    if sub.is_direct_mode():\n        # 'sub' is now considered to be clean; therefore it's not reported as\n        # modified upwards\n        # This is consistent in a way, but surprising in another ...\n        pass\n    else:\n        eq_(stat, ar.get_status())\n    sub.commit(msg=\"added file not_tracked\")\n    # 'submod/' still modified when looked at from above:\n    eq_(stat, ar.get_status())\n\n    ar.add('submod', git=True)\n    ar.commit(msg=\"submodule modified\", files='submod')\n    stat['modified'].remove('submod/')\n    eq_(stat, ar.get_status())\n\n    # clean again:\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n\n    ar.add('first', git=True)\n    ar.add(opj('sub', 'third'))\n    ar.add('fifth')\n    sync_wrapper()\n\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # mixed annexed/not-annexed files ATm can't be committed with explicitly\n        # given paths in v6\n        # See:\n        # http://git-annex.branchable.com/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6\n        ar.commit()\n    else:\n        super(AnnexRepo, ar).commit(files=['first', 'fifth', opj('sub', 'third'), 'second'])\n    eq_(stat, ar.get_status())\n\n    # remove a file in annex:\n    ar.remove('fifth')\n    stat['deleted'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # remove a file in git:\n    ar.remove('first')\n    stat['deleted'].append('first')\n    eq_(stat, ar.get_status())\n\n    # remove a submodule:\n    # rm; git rm; git commit\n    sub.precommit()  # Do precommit so there are not active batched processes etc\n    from datalad.utils import rmtree\n    rmtree(opj(ar.path, 'submod'))\n    stat['deleted'].append('submod')\n    eq_(stat, ar.get_status())\n    # recreate an empty mountpoint, since we currently do it in uninstall:\n    os.makedirs(opj(ar.path, 'submod'))\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n    ar.remove('submod')\n    # TODO: Why the difference?\n    stat['deleted'].append('submod')\n    stat['modified'].append('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # Note: Here again we need to use annex-proxy; This contradicts the addition\n    # of the very same submodule, which we needed to commit via\n    # -c core.bare=False instead. Otherwise the very same failure happens.\n    # Just vice versa. See above where 'submod' is added.\n    ar.commit(\"submod removed\", files=['submod', '.gitmodules'])\n    stat['modified'].remove('.gitmodules')\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_status(path, path2):\n\n    ar = AnnexRepo(path, create=True)\n    _test_status(ar)\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # in case of v6 have a second run with adjusted branch feature:\n        ar2 = AnnexRepo(path2, create=True)\n        ar2.commit(msg=\"empty commit to create branch 'master'\",\n                   options=['--allow-empty'])\n        ar2._run_annex_command('adjust', annex_options=['--unlock'])\n        _test_status(ar2)\n\n\n\n# TODO: test dirty\n# TODO: GitRep.dirty\n# TODO: test/utils ok_clean_git\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_set_remote_url(path):\n\n    ar = AnnexRepo(path, create=True)\n    ar.add_remote('some', 'http://example.com/.git')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://example.com/.git')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n    # change url:\n    ar.set_remote_url('some', 'http://believe.it')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://believe.it')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n\n    # set push url:\n    ar.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    assert_equal(ar.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n    assert_in('remote.some.annexurl', ar.config.keys())\n    assert_equal(ar.config['remote.some.annexurl'],\n                 'ssh://whatever.ru')\n\n\n@with_tempfile(mkdir=True)\ndef test_wanted(path):\n    ar = AnnexRepo(path, create=True)\n    eq_(ar.get_preferred_content('wanted'), None)\n    # test samples with increasing \"trickiness\"\n    for v in (\"standard\",\n              \"include=*.nii.gz or include=*.nii\",\n              \"exclude=archive/* and (include=*.dat or smallerthan=2b)\"\n              ):\n        ar.set_preferred_content('wanted', expr=v)\n        eq_(ar.get_preferred_content('wanted'), v)\n    # give it some file so clone/checkout works without hiccups\n    create_tree(ar.path, {'1.dat': 'content'})\n    ar.add('1.dat')\n    ar.commit(msg=\"blah\")\n    # make a clone and see if all cool there\n    # intentionally clone as pure Git and do not annex init so to see if we\n    # are ignoring crummy log msgs\n    ar1_path = ar.path + '_1'\n    GitRepo.clone(ar.path, ar1_path)\n    ar1 = AnnexRepo(ar1_path, init=False)\n    eq_(ar1.get_preferred_content('wanted'), None)\n    eq_(ar1.get_preferred_content('wanted', 'origin'), v)\n    ar1.set_preferred_content('wanted', expr='standard')\n    eq_(ar1.get_preferred_content('wanted'), 'standard')\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_metadata(path):\n    # prelude\n    ar = AnnexRepo(path, create=True)\n    create_tree(\n        path,\n        {\n            'up.dat': 'content',\n            'd o\"w n': {\n                'd o w n.dat': 'lowcontent'\n            }\n        })\n    ar.add('.', git=False)\n    ar.commit('content')\n    ok_clean_git(path)\n    # fugue\n    # doesn't do anything if there is nothing to do\n    ar.set_metadata('up.dat')\n    eq_([], list(ar.get_metadata(None)))\n    eq_([], list(ar.get_metadata('')))\n    eq_([], list(ar.get_metadata([])))\n    eq_({'up.dat': {}}, dict(ar.get_metadata('up.dat')))\n    # basic invocation\n    eq_(1, len(list(ar.set_metadata(\n        'up.dat',\n        reset={'mike': 'awesome'},\n        add={'tag': 'awesome'},\n        remove={'tag': 'awesome'},  # cancels prev, just to use it\n        init={'virgin': 'true'},\n        purge=['nothere']))))\n    # no timestamps by default\n    md = dict(ar.get_metadata('up.dat'))\n    deq_({'up.dat': {\n        'virgin': ['true'],\n        'mike': ['awesome']}},\n        md)\n    # matching timestamp entries for all keys\n    md_ts = dict(ar.get_metadata('up.dat', timestamps=True))\n    for k in md['up.dat']:\n        assert_in('{}-lastchanged'.format(k), md_ts['up.dat'])\n    assert_in('lastchanged', md_ts['up.dat'])\n    # recursive needs a flag\n    assert_raises(CommandError, list, ar.set_metadata('.', purge=['virgin']))\n    list(ar.set_metadata('.', purge=['virgin'], recursive=True))\n    deq_({'up.dat': {\n        'mike': ['awesome']}},\n        dict(ar.get_metadata('up.dat')))\n    # Use trickier tags (spaces, =)\n    list(ar.set_metadata('.', reset={'tag': 'one and= '}, purge=['mike'], recursive=True))\n    playfile = opj('d o\"w n', 'd o w n.dat')\n    target = {\n        'up.dat': {\n            'tag': ['one and= ']},\n        playfile: {\n            'tag': ['one and= ']}}\n    deq_(target, dict(ar.get_metadata('.')))\n    # incremental work like a set\n    list(ar.set_metadata(playfile, add={'tag': 'one and= '}))\n    deq_(target, dict(ar.get_metadata('.')))\n    list(ar.set_metadata(playfile, add={'tag': ' two'}))\n    # returned values are sorted\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    # init honor prior values\n    list(ar.set_metadata(playfile, init={'tag': 'three'}))\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    list(ar.set_metadata(playfile, remove={'tag': ' two'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # remove non-existing doesn't error and doesn't change anything\n    list(ar.set_metadata(playfile, remove={'ether': 'best'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # add works without prior existence\n    list(ar.set_metadata(playfile, add={'novel': 'best'}))\n    eq_(['best'], dict(ar.get_metadata(playfile))[playfile]['novel'])\n\n\n@with_tree(tree={'file.txt': 'content'})\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_batched_and_set_metadata(path, url, dest):\n    ar = AnnexRepo(dest, create=True)\n    fname = \"file.txt\"\n    ar.add_url_to_file(fname, urljoin(url, fname), batch=True)\n    list(ar.set_metadata(fname, init={\"number\": \"one\"}))\n    eq_([\"one\"], dict(ar.get_metadata(fname))[fname][\"number\"])\n\n\n@with_tempfile(mkdir=True)\ndef test_change_description(path):\n    # prelude\n    ar = AnnexRepo(path, create=True, description='some')\n    eq_(ar.get_description(), 'some')\n    # try change it\n    ar = AnnexRepo(path, create=False, init=True, description='someother')\n    # this doesn't cut the mustard, still old\n    eq_(ar.get_description(), 'some')\n    # need to resort to \"internal\" helper\n    ar._init(description='someother')\n    eq_(ar.get_description(), 'someother')\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_corresponding_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we should be on master or a corresponding branch like annex/direct/master\n    # respectively if ran in direct mode build.\n    # We want to get 'master' in any case\n    eq_('master', ar.get_corresponding_branch())\n\n    # special case v6 adjusted branch is not provided by a dedicated build:\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        # as above, we still want to get 'master', while being on\n        # 'adjusted/master(unlocked)'\n        eq_('adjusted/master(unlocked)', ar.get_active_branch())\n        eq_('master', ar.get_corresponding_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_tracking_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we want the relation to original branch, especially in direct mode\n    # or even in v6 adjusted branch\n    eq_(('origin', 'refs/heads/master'), ar.get_tracking_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_is_managed_branch(path):\n\n    ar = AnnexRepo(path)\n\n    if ar.is_direct_mode():\n        ok_(ar.is_managed_branch())\n    else:\n        # ATM only direct mode and v6 adjusted branches should return True.\n        # Adjusted branch requires a call of git-annex-adjust and shouldn't\n        # be the state of a fresh clone\n        ok_(not ar.is_managed_branch())\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        ok_(ar.is_managed_branch())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_AnnexRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(AnnexRepo, path, store)\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_is_not_special(path):\n    ar = AnnexRepo(path, create=True)\n    # doesn't exist -- we fail by default\n    assert_raises(RemoteNotAvailableError, ar.is_special_annex_remote, \"fake\")\n    assert_false(ar.is_special_annex_remote(\"fake\", check_if_known=False))\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_dates(path):\n    ar = AnnexRepo(path, create=True, fake_dates=True)\n    timestamp = ar.config.obtain(\"datalad.fake-dates-start\") + 1\n    # Commits from the \"git annex init\" call are one second ahead.\n    for commit in ar.get_branch_commits(\"git-annex\"):\n        eq_(timestamp, commit.committed_date)\n    assert_in(\"timestamp={}s\".format(timestamp),\n              ar.repo.git.cat_file(\"blob\", \"git-annex:uuid.log\"))\n\n\ndef test_get_size_from_perc_complete():\n    f = _get_size_from_perc_complete\n    eq_(f(0, 0), 0)\n    eq_(f(0, '0'), 0)\n    eq_(f(100, '0'), 0)  # we do not know better\n    eq_(f(1, '1'), 100)\n    # with no percentage info, we don't know better either:\n    eq_(f(1, ''), 0)\n\n\n# to prevent regression\n# http://git-annex.branchable.com/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/\n# It is disabled because is not per se relevant to DataLad since we do not\n# Since we invoke from the top of the repo, we do not hit it,\n# but thought to leave it around if we want to enforce/test system-wide git being\n# compatible with annex for v6 mode\n@with_tempfile(mkdir=True)\ndef _test_add_under_subdir(path):\n    ar = AnnexRepo(path, create=True, version=6)\n    gr = GitRepo(path)  # \"Git\" view over the repository, so we force \"git add\"\n    subdir = opj(path, 'sub')\n    subfile = opj('sub', 'empty')\n    # os.mkdir(subdir)\n    create_tree(subdir, {'empty': ''})\n    runner = Runner(cwd=subdir)\n    with chpwd(subdir):\n        runner(['git', 'add', 'empty'])  # should add sucesfully\n        # gr.commit('important') #\n        runner(['git', 'commit', '-m', 'important'])\n        ar.is_under_annex(subfile)\n",
            "header": {
                "file": {
                    "path": "datalad/support/tests/test_annexrepo.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "a5fb9571b",
                "new_commit_hash": "711b7d9cc",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/support/tests/test_annexrepo.py"
            },
            "plus_file": {
                "path": "b/datalad/support/tests/test_annexrepo.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 2358,
                            "length": 6
                        },
                        "new_range": {
                            "start": 2358,
                            "length": 8
                        },
                        "section": "def test_get_size_from_perc_complete():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    eq_(f(0, '0'), 0)",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(f(100, '0'), 0)  # we do not know better",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(f(1, '1'), 100)",
                                "type": "context"
                            },
                            {
                                "content": "    # with no percentage info, we don't know better either:",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(f(1, ''), 0)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "# to prevent regression",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/tests/test_annexrepo.py",
                            "type": "function",
                            "name": "test_get_size_from_perc_complete",
                            "content": "def test_get_size_from_perc_complete():\n    f = _get_size_from_perc_complete\n    eq_(f(0, 0), 0)\n    eq_(f(0, '0'), 0)\n    eq_(f(100, '0'), 0)  # we do not know better\n    eq_(f(1, '1'), 100)\n    # with no percentage info, we don't know better either:\n    eq_(f(1, ''), 0)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 2355,
                            "end_lineno": 2362,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 2379,
                            "length": 4
                        },
                        "new_range": {
                            "start": 2381,
                            "length": 4
                        },
                        "section": "def _test_add_under_subdir(path):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        runner(['git', 'add', 'empty'])  # should add sucesfully",
                                "type": "context"
                            },
                            {
                                "content": "        # gr.commit('important') #",
                                "type": "context"
                            },
                            {
                                "content": "        runner(['git', 'commit', '-m', 'important'])",
                                "type": "context"
                            },
                            {
                                "content": "        ar.is_under_annex(subfile)",
                                "type": "deleted"
                            },
                            {
                                "content": "No newline at end of file",
                                "type": "note"
                            },
                            {
                                "content": "        ar.is_under_annex(subfile)",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/support/tests/test_annexrepo.py",
                            "type": "function",
                            "name": "_test_add_under_subdir",
                            "content": "def _test_add_under_subdir(path):\n    ar = AnnexRepo(path, create=True, version=6)\n    gr = GitRepo(path)  # \"Git\" view over the repository, so we force \"git add\"\n    subdir = opj(path, 'sub')\n    subfile = opj('sub', 'empty')\n    # os.mkdir(subdir)\n    create_tree(subdir, {'empty': ''})\n    runner = Runner(cwd=subdir)\n    with chpwd(subdir):\n        runner(['git', 'add', 'empty'])  # should add sucesfully\n        # gr.commit('important') #\n        runner(['git', 'commit', '-m', 'important'])\n        ar.is_under_annex(subfile)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 2370,
                            "end_lineno": 2382,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9^",
    "new_commit_hash": "b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9",
    "commit_message": "Merge pull request #2891 from bpoldrack/bf-addurl-progress\n\nBF: Don't crash on no reported percentage",
    "commit_date": "2018-10-03T18:40:01+02:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test implementation of class AnnexRepo\n\n\"\"\"\n\nfrom datalad.tests.utils import known_failure_v6\n\nimport logging\nfrom functools import partial\nimport os\nfrom os import mkdir\nfrom os.path import join as opj\nfrom os.path import basename\nfrom os.path import realpath\nfrom os.path import relpath\nfrom os.path import curdir\nfrom os.path import pardir\nfrom os.path import exists\nfrom shutil import copyfile\nfrom nose.tools import assert_not_is_instance\n\nfrom six.moves.urllib.parse import urljoin\nfrom six.moves.urllib.parse import urlsplit\n\nimport git\nfrom git import GitCommandError\nfrom mock import patch\nimport gc\n\nfrom datalad.cmd import Runner\n\nfrom datalad.support.external_versions import external_versions\n\nfrom datalad.support.sshconnector import get_connection_hash\n\nfrom datalad.utils import on_windows\nfrom datalad.utils import chpwd\nfrom datalad.utils import rmtree\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import unlink\n\nfrom datalad.tests.utils import ignore_nose_capturing_stdout\nfrom datalad.tests.utils import assert_cwd_unchanged\nfrom datalad.tests.utils import with_testrepos\nfrom datalad.tests.utils import with_tempfile\nfrom datalad.tests.utils import with_tree\nfrom datalad.tests.utils import create_tree\nfrom datalad.tests.utils import with_batch_direct\nfrom datalad.tests.utils import assert_dict_equal as deq_\nfrom datalad.tests.utils import assert_is_instance\nfrom datalad.tests.utils import assert_false\nfrom datalad.tests.utils import assert_in\nfrom datalad.tests.utils import assert_is\nfrom datalad.tests.utils import assert_not_in\nfrom datalad.tests.utils import assert_re_in\nfrom datalad.tests.utils import assert_raises\nfrom datalad.tests.utils import assert_not_equal\nfrom datalad.tests.utils import assert_equal\nfrom datalad.tests.utils import assert_true\nfrom datalad.tests.utils import eq_\nfrom datalad.tests.utils import ok_\nfrom datalad.tests.utils import ok_git_config_not_empty\nfrom datalad.tests.utils import ok_annex_get\nfrom datalad.tests.utils import ok_clean_git\nfrom datalad.tests.utils import ok_file_has_content\nfrom datalad.tests.utils import swallow_logs\nfrom datalad.tests.utils import swallow_outputs\nfrom datalad.tests.utils import local_testrepo_flavors\nfrom datalad.tests.utils import serve_path_via_http\nfrom datalad.tests.utils import get_most_obscure_supported_name\nfrom datalad.tests.utils import OBSCURE_FILENAME\nfrom datalad.tests.utils import SkipTest\nfrom datalad.tests.utils import skip_ssh\nfrom datalad.tests.utils import find_files\n\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.exceptions import CommandNotAvailableError\nfrom datalad.support.exceptions import FileNotInRepositoryError\nfrom datalad.support.exceptions import FileNotInAnnexError\nfrom datalad.support.exceptions import FileInGitError\nfrom datalad.support.exceptions import OutOfSpaceError\nfrom datalad.support.exceptions import RemoteNotAvailableError\nfrom datalad.support.exceptions import OutdatedExternalDependency\nfrom datalad.support.exceptions import MissingExternalDependency\nfrom datalad.support.exceptions import InsufficientArgumentsError\nfrom datalad.support.exceptions import AnnexBatchCommandError\nfrom datalad.support.exceptions import IncompleteResultsError\n\nfrom datalad.support.gitrepo import GitRepo\n\n# imports from same module:\nfrom datalad.support.annexrepo import (\n    AnnexRepo,\n    ProcessAnnexProgressIndicators,\n    _get_size_from_perc_complete,\n)\nfrom datalad.support.tests.utils import check_repo_deals_with_inode_change\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_instance_from_clone(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(dst, '.git', 'annex')))\n\n    # do it again should raise GitCommandError since git will notice\n    # there's already a git-repo at that path and therefore can't clone to `dst`\n    with swallow_logs(new_level=logging.WARN) as cm:\n        assert_raises(GitCommandError, AnnexRepo.clone, src, dst)\n        if git.__version__ != \"1.0.2\" and git.__version__ != \"2.0.5\":\n            assert(\"already exists\" in cm.out)\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_instance_from_existing(path):\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_instance_brand_new(path):\n\n    GitRepo(path)\n    assert_raises(RuntimeError, AnnexRepo, path, create=False)\n\n    ar = AnnexRepo(path)\n    assert_is_instance(ar, AnnexRepo, \"AnnexRepo was not created.\")\n    ok_(os.path.exists(os.path.join(path, '.git')))\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_crippled_filesystem(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    # fake git-annex entries in .git/config:\n    writer = ar.repo.config_writer()\n    writer.set_value(\"annex\", \"crippledfilesystem\", True)\n    writer.release()\n    ok_(ar.is_crippled_fs())\n    writer.set_value(\"annex\", \"crippledfilesystem\", False)\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n    # since we can't remove the entry, just rename it to fake its absence:\n    writer.rename_section(\"annex\", \"removed\")\n    writer.set_value(\"annex\", \"something\", \"value\")\n    writer.release()\n    assert_false(ar.is_crippled_fs())\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\ndef test_AnnexRepo_is_direct_mode(path):\n\n    ar = AnnexRepo(path)\n    eq_(ar.config.getbool(\"annex\", \"direct\", False),\n        ar.is_direct_mode())\n\n\n@with_tempfile()\ndef test_AnnexRepo_is_direct_mode_gitrepo(path):\n    repo = GitRepo(path, create=True)\n    # artificially make .git/annex so no annex section gets initialized\n    # in .git/config.  We did manage somehow to make this happen (via publish)\n    # but didn't reproduce yet, so just creating manually\n    mkdir(opj(repo.path, '.git', 'annex'))\n    ar = AnnexRepo(path, init=False, create=False)\n    # It is unlikely though that annex would be in direct mode (requires explicit)\n    # annex magic, without having annex section under .git/config\n    dm = ar.is_direct_mode()\n\n    if ar.is_crippled_fs() or on_windows:\n        ok_(dm)\n    else:\n        assert_false(dm)\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*')\n@with_tempfile\ndef test_AnnexRepo_set_direct_mode(src, dst):\n\n    ar = AnnexRepo.clone(src, dst)\n\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available:\n        assert_raises(CommandError, ar.set_direct_mode, True)\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n\n    ar.set_direct_mode(True)\n    ok_(ar.is_direct_mode(), \"Switching to direct mode failed.\")\n    if ar.is_crippled_fs():\n        assert_raises(CommandNotAvailableError, ar.set_direct_mode, False)\n        ok_(\n            ar.is_direct_mode(),\n            \"Indirect mode on crippled fs detected. Shouldn't be possible.\")\n    else:\n        ar.set_direct_mode(False)\n        assert_false(ar.is_direct_mode(), \"Switching to indirect mode failed.\")\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_annex_proxy(src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path)\n    if ar.config.getint(\"annex\", \"version\") >= 6:\n        # there's no direct mode available and therefore no 'annex proxy':\n        assert_raises(CommandError, ar.proxy, ['git', 'status'])\n        raise SkipTest(\"Test not applicable in repository version >= 6\")\n    ar.set_direct_mode(True)\n\n    # annex proxy raises in indirect mode:\n    try:\n        ar.set_direct_mode(False)\n        assert_raises(CommandNotAvailableError, ar.proxy, ['git', 'status'])\n    except CommandNotAvailableError:\n        # we can't switch to indirect\n        pass\n\n\n@assert_cwd_unchanged\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_key(src, annex_path):\n\n    ar = AnnexRepo.clone(src, annex_path)\n\n    # test-annex.dat should return the correct key:\n    eq_(\n        ar.get_file_key(\"test-annex.dat\"),\n        'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat')\n\n    # and should take a list with an empty string as result, if a file wasn't\n    # in annex:\n    eq_(\n        ar.get_file_key([\"filenotpresent.wtf\", \"test-annex.dat\"]),\n        ['', 'SHA256E-s28--2795fb26981c5a687b9bf44930cc220029223f472cea0f0b17274f4473181e7b.dat']\n    )\n\n    # test.dat is actually in git\n    # should raise Exception; also test for polymorphism\n    assert_raises(IOError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileNotInAnnexError, ar.get_file_key, \"test.dat\")\n    assert_raises(FileInGitError, ar.get_file_key, \"test.dat\")\n\n    # filenotpresent.wtf doesn't even exist\n    assert_raises(IOError, ar.get_file_key, \"filenotpresent.wtf\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_get_outofspace(annex_path):\n    ar = AnnexRepo(annex_path, create=True)\n\n    def raise_cmderror(*args, **kwargs):\n        raise CommandError(\n            cmd=\"whatever\",\n            stderr=\"junk around not enough free space, need 905.6 MB more and after\"\n        )\n\n    with patch.object(AnnexRepo, '_run_annex_command', raise_cmderror) as cma, \\\n            assert_raises(OutOfSpaceError) as cme:\n        ar.get(\"file\")\n    exc = cme.exception\n    eq_(exc.sizemore_msg, '905.6 MB')\n    assert_re_in(\".*annex (find|get). needs 905.6 MB more\", str(exc))\n\n\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_AnnexRepo_get_remote_na(path):\n    ar = AnnexRepo(path)\n\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', options=[\"--from=NotExistingRemote\"])\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n    # and similar one whenever invoking with remote parameter\n    with assert_raises(RemoteNotAvailableError) as cme:\n        ar.get('test-annex.dat', remote=\"NotExistingRemote\")\n    eq_(cme.exception.remote, \"NotExistingRemote\")\n\n\n# 1 is enough to test file_has_content\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_file_has_content(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n    testfiles = [\"test-annex.dat\", \"test.dat\"]\n\n    eq_(ar.file_has_content(testfiles), [False, False])\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.file_has_content(testfiles, batch=batch), [True, False])\n    eq_(ar.file_has_content(testfiles[:1], batch=batch), [True])\n\n    eq_(ar.file_has_content(testfiles + [\"bogus.txt\"], batch=batch),\n        [True, False, False])\n\n    assert_false(ar.file_has_content(\"bogus.txt\", batch=batch))\n    ok_(ar.file_has_content(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.file_has_content([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n# 1 is enough to test\n@with_batch_direct\n@with_testrepos('.*annex.*', flavors=['local'], count=1)\n@with_tempfile\ndef test_AnnexRepo_is_under_annex(batch, direct, src, annex_path):\n    ar = AnnexRepo.clone(src, annex_path, direct=direct)\n\n    with open(opj(annex_path, 'not-committed.txt'), 'w') as f:\n        f.write(\"aaa\")\n\n    testfiles = [\"test-annex.dat\", \"not-committed.txt\", \"INFO.txt\"]\n    # wouldn't change\n    target_value = [True, False, False]\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n\n    ok_annex_get(ar, \"test-annex.dat\")\n    eq_(ar.is_under_annex(testfiles, batch=batch), target_value)\n    eq_(ar.is_under_annex(testfiles[:1], batch=batch), target_value[:1])\n    eq_(ar.is_under_annex(testfiles[1:], batch=batch), target_value[1:])\n\n    eq_(ar.is_under_annex(testfiles + [\"bogus.txt\"], batch=batch),\n                 target_value + [False])\n\n    assert_false(ar.is_under_annex(\"bogus.txt\", batch=batch))\n    ok_(ar.is_under_annex(\"test-annex.dat\", batch=batch))\n\n    if not direct:  # There's no unlock in direct mode.\n        ar.unlock([\"test-annex.dat\"])\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [ar.config.get(\"annex.version\") == \"6\"])\n        with open(opj(annex_path, \"test-annex.dat\"), \"a\") as ofh:\n            ofh.write(\"more\")\n        eq_(ar.is_under_annex([\"test-annex.dat\"], batch=batch),\n            [False])\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_web_remote(sitepath, siteurl, dst):\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = '%s_about.txt' % url_file_prefix\n    testfile2 = '%s_about2.txt' % url_file_prefix\n    testfile3 = opj('d', 'sub.txt')\n\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl])\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # output='full'\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull), set(l))  # the same entries\n    non_web_remote = l[1 - l.index(ar.WEB_UUID)]\n    assert_in('urls', lfull[non_web_remote])\n    eq_(lfull[non_web_remote]['urls'], [])\n    assert_not_in('uuid', lfull[ar.WEB_UUID])  # no uuid in the records\n    eq_(lfull[ar.WEB_UUID]['urls'], [testurl])\n\n    # --all and --key are incompatible\n    assert_raises(CommandError, ar.whereis, [], options='--all', output='full', key=True)\n\n    # output='descriptions'\n    ldesc = ar.whereis(testfile, output='descriptions')\n    eq_(set(ldesc), set([v['description'] for v in lfull.values()]))\n\n    # info w/ and w/o fast mode\n    for fast in [True, False]:\n        info = ar.info(testfile, fast=fast)\n        eq_(info['size'], 14)\n        assert(info['key'])  # that it is there\n        info_batched = ar.info(testfile, batch=True, fast=fast)\n        eq_(info, info_batched)\n        # while at it ;)\n        with swallow_outputs() as cmo:\n            eq_(ar.info('nonexistent', batch=False), None)\n            eq_(ar.info('nonexistent-batch', batch=True), None)\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n            ar.precommit()  # to stop all the batched processes for swallow_outputs\n\n    # annex repo info\n    repo_info = ar.repo_info(fast=False)\n    eq_(repo_info['local annex size'], 14)\n    eq_(repo_info['backend usage'], {'SHA256E': 1})\n    # annex repo info in fast mode\n    repo_info_fast = ar.repo_info(fast=True)\n    # doesn't give much testable info, so just comparing a subset for match with repo_info info\n    eq_(repo_info_fast['semitrusted repositories'], repo_info['semitrusted repositories'])\n    #import pprint; pprint.pprint(repo_info)\n\n    # remove the remote\n    ar.rm_url(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_not_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n\n    # now only 1 copy; drop should fail\n    res = ar.drop(testfile)\n    eq_(res['command'], 'drop')\n    eq_(res['success'], False)\n    assert_in('adjust numcopies', res['note'])\n\n    # read the url using different method\n    ar.add_url_to_file(testfile, testurl)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 2)\n    ok_(ar.file_has_content(testfile))\n\n    # 2 known copies now; drop should succeed\n    ar.drop(testfile)\n    l = ar.whereis(testfile)\n    assert_in(ar.WEB_UUID, l)\n    eq_(len(l), 1)\n    assert_false(ar.file_has_content(testfile))\n    lfull = ar.whereis(testfile, output='full')\n    assert_not_in(non_web_remote, lfull) # not present -- so not even listed\n\n    # multiple files/urls\n    # get the file from remote\n    with swallow_outputs() as cmo:\n        ar.add_urls([testurl2])\n\n    # TODO: if we ask for whereis on all files, we should get for all files\n    lall = ar.whereis('.')\n    eq_(len(lall), 2)\n    for e in lall:\n        assert(isinstance(e, list))\n    # but we don't know which one for which file. need a 'full' one for that\n    lall_full = ar.whereis('.', output='full')\n    ok_(ar.file_has_content(testfile2))\n    ok_(lall_full[testfile2][non_web_remote]['here'])\n    eq_(set(lall_full), {testfile, testfile2})\n\n    # add a bogus 2nd url to testfile\n\n    someurl = \"http://example.com/someurl\"\n    ar.add_url_to_file(testfile, someurl, options=['--relaxed'])\n    lfull = ar.whereis(testfile, output='full')\n    eq_(set(lfull[ar.WEB_UUID]['urls']), {testurl, someurl})\n\n    # and now test with a file in subdirectory\n    subdir = opj(dst, 'd')\n    os.mkdir(subdir)\n    with swallow_outputs() as cmo:\n        ar.add_url_to_file(testfile3, url=testurl3)\n    ok_file_has_content(opj(dst, testfile3), 'more stuff')\n    eq_(set(ar.whereis(testfile3)), {ar.WEB_UUID, non_web_remote})\n    eq_(set(ar.whereis(testfile3, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n\n    # and if we ask for both files\n    info2 = ar.info([testfile, testfile3])\n    eq_(set(info2), {testfile, testfile3})\n    eq_(info2[testfile3]['size'], 10)\n\n    full = ar.whereis([], options='--all', output='full')\n    eq_(len(full.keys()), 3)  # we asked for all files -- got 3 keys\n    assert_in(ar.WEB_UUID, full['SHA256E-s10--a978713ea759207f7a6f9ebc9eaebd1b40a69ae408410ddf544463f6d33a30e1.txt'])\n\n    # which would work even if we cd to that subdir, but then we should use explicit curdir\n    with chpwd(subdir):\n        cur_subfile = opj(curdir, 'sub.txt')\n        eq_(set(ar.whereis(cur_subfile)), {ar.WEB_UUID, non_web_remote})\n        eq_(set(ar.whereis(cur_subfile, output='full').keys()), {ar.WEB_UUID, non_web_remote})\n        testfiles = [cur_subfile, opj(pardir, testfile)]\n        info2_ = ar.info(testfiles)\n        # Should maintain original relative file names\n        eq_(set(info2_), set(testfiles))\n        eq_(info2_[cur_subfile]['size'], 10)\n\n\n@with_tree(tree={\"a.txt\": \"a\",\n                 \"b\": \"b\",\n                 OBSCURE_FILENAME: \"c\",\n                 \"subdir\": {\"d\": \"d\", \"e\": \"e\"}})\ndef test_find_batch_equivalence(path):\n    ar = AnnexRepo(path)\n    files = [\"a.txt\", \"b\", OBSCURE_FILENAME]\n    ar.add(files + [\"subdir\"])\n    ar.commit(\"add files\")\n    query = [\"not-there\"] + files\n    expected = {f: f for f in files}\n    expected.update({\"not-there\": \"\"})\n    eq_(expected, ar.find(query, batch=True))\n    eq_(expected, ar.find(query))\n    # If we give a subdirectory, we split that output.\n    eq_(set(ar.find([\"subdir\"])[\"subdir\"]), {\"subdir/d\", \"subdir/e\"})\n    eq_(ar.find([\"subdir\"]), ar.find([\"subdir\"], batch=True))\n\n\n@with_tempfile(mkdir=True)\ndef test_repo_info(path):\n    repo = AnnexRepo(path)\n    info = repo.repo_info()  # works in empty repo without crashing\n    eq_(info['local annex size'], 0)\n    eq_(info['size of annexed files in working tree'], 0)\n\n    def get_custom(custom={}):\n        \"\"\"Need a helper since repo_info modifies in place so we should generate\n        new each time\n        \"\"\"\n        custom_json = {\n            'available local disk space': 'unknown',\n            'size of annexed files in working tree': \"0\",\n            'success': True,\n            'command': 'info',\n        }\n        if custom:\n            custom_json.update(custom)\n        return [custom_json]\n\n    with patch.object(\n            repo, '_run_annex_command_json',\n            return_value=get_custom()):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], None)\n\n    with patch.object(\n        repo, '_run_annex_command_json',\n        return_value=get_custom({\n            \"available local disk space\": \"19193986496 (+100000 reserved)\"})):\n        info = repo.repo_info()\n        eq_(info['available local disk space'], 19193986496)\n\n\n@with_testrepos('.*annex.*', flavors=['local', 'network'])\n@with_tempfile\ndef test_AnnexRepo_migrating_backends(src, dst):\n    ar = AnnexRepo.clone(src, dst, backend='MD5')\n    eq_(ar.default_backends, ['MD5'])\n    # GitPython has a bug which causes .git/config being wiped out\n    # under Python3, triggered by collecting its config instance I guess\n    gc.collect()\n    ok_git_config_not_empty(ar)  # Must not blow, see https://github.com/gitpython-developers/GitPython/issues/333\n\n    filename = get_most_obscure_supported_name()\n    filename_abs = os.path.join(dst, filename)\n    f = open(filename_abs, 'w')\n    f.write(\"What to write?\")\n    f.close()\n\n    ar.add(filename, backend='MD5')\n    eq_(ar.get_file_backend(filename), 'MD5')\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n\n    # migrating will only do, if file is present\n    ok_annex_get(ar, 'test-annex.dat')\n\n    if ar.is_direct_mode():\n        # No migration in direct mode\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat')\n    else:\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n        ar.migrate_backend('test-annex.dat')\n        eq_(ar.get_file_backend('test-annex.dat'), 'MD5')\n\n        ar.migrate_backend('', backend='SHA1')\n        eq_(ar.get_file_backend(filename), 'SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n\n\ntree1args = dict(\n    tree=(\n        ('firstfile', 'whatever'),\n        ('secondfile', 'something else'),\n        ('remotefile', 'pretends to be remote'),\n        ('faraway', 'incredibly remote')),\n)\n\n# keys for files if above tree is generated and added to annex with MD5E backend\ntree1_md5e_keys = {\n    'firstfile': 'MD5E-s8--008c5926ca861023c1d2a36653fd88e2',\n    'faraway': 'MD5E-s17--5b849ed02f914d3bbb5038fe4e3fead9',\n    'secondfile': 'MD5E-s14--6c7ba9c5a141421e1c03cb9807c97c74',\n    'remotefile': 'MD5E-s21--bf7654b3de20d5926d407ea7d913deb0'\n}\n\n\n@with_tree(**tree1args)\ndef __test_get_md5s(path):\n    # was used just to generate above dict\n    annex = AnnexRepo(path, init=True, backend='MD5E')\n    files = [basename(f) for f in find_files('.*', path)]\n    annex.add(files)\n    annex.commit()\n    print({f: annex.get_file_key(f) for f in files})\n\n\n@with_batch_direct\n@with_tree(**tree1args)\ndef test_dropkey(batch, direct, path):\n    kw = {'batch': batch}\n    annex = AnnexRepo(path, init=True, backend='MD5E', direct=direct)\n    files = list(tree1_md5e_keys)\n    annex.add(files)\n    annex.commit()\n    # drop one key\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # drop multiple\n    annex.drop_key([tree1_md5e_keys[f] for f in files[1:3]], **kw)\n    # drop already dropped -- should work as well atm\n    # https://git-annex.branchable.com/bugs/dropkey_--batch_--json_--force_is_always_succesfull\n    annex.drop_key(tree1_md5e_keys[files[0]], **kw)\n    # and a mix with already dropped or not\n    annex.drop_key(list(tree1_md5e_keys.values()), **kw)\n\n\n@with_tree(**tree1args)\n@serve_path_via_http()\ndef test_AnnexRepo_backend_option(path, url):\n    ar = AnnexRepo(path, backend='MD5')\n\n    # backend recorded in .gitattributes\n    eq_(ar.get_gitattributes('.')['.']['annex.backend'], 'MD5')\n\n    ar.add('firstfile', backend='SHA1')\n    ar.add('secondfile')\n    eq_(ar.get_file_backend('firstfile'), 'SHA1')\n    eq_(ar.get_file_backend('secondfile'), 'MD5')\n\n    with swallow_outputs() as cmo:\n        # must be added under different name since annex 20160114\n        ar.add_url_to_file('remotefile2', url + 'remotefile', backend='SHA1')\n    eq_(ar.get_file_backend('remotefile2'), 'SHA1')\n\n    with swallow_outputs() as cmo:\n        ar.add_urls([url + 'faraway'], backend='SHA1')\n    # TODO: what's the annex-generated name of this?\n    # For now, workaround:\n    ok_(ar.get_file_backend(f) == 'SHA1'\n        for f in ar.get_indexed_files() if 'faraway' in f)\n\n\n@with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n@with_tempfile\ndef test_AnnexRepo_get_file_backend(src, dst):\n    #init local test-annex before cloning:\n    AnnexRepo(src)\n\n    ar = AnnexRepo.clone(src, dst)\n\n    eq_(ar.get_file_backend('test-annex.dat'), 'SHA256E')\n    if not ar.is_direct_mode():\n        # no migration in direct mode\n        ok_annex_get(ar, 'test-annex.dat', network=False)\n        ar.migrate_backend('test-annex.dat', backend='SHA1')\n        eq_(ar.get_file_backend('test-annex.dat'), 'SHA1')\n    else:\n        assert_raises(CommandNotAvailableError, ar.migrate_backend,\n                      'test-annex.dat', backend='SHA1')\n\n\n@with_tempfile\ndef test_AnnexRepo_always_commit(path):\n\n    repo = AnnexRepo(path)\n    runner = Runner(cwd=path)\n    file1 = get_most_obscure_supported_name() + \"_1\"\n    file2 = get_most_obscure_supported_name() + \"_2\"\n    with open(opj(path, file1), 'w') as f:\n        f.write(\"First file.\")\n    with open(opj(path, file2), 'w') as f:\n        f.write(\"Second file.\")\n\n    # always_commit == True is expected to be default\n    repo.add(file1)\n\n    # Now git-annex log should show the addition:\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 1)\n    assert_in(file1, out_list[0])\n    # check git log of git-annex branch:\n    # expected: initial creation, update (by annex add) and another\n    # update (by annex log)\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = False\n    repo.add(file2)\n\n    # No additional git commit:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 3)\n\n    repo.always_commit = True\n\n    # Still one commit only in git-annex log,\n    # but 'git annex log' was called when always_commit was true again,\n    # so it should commit the addition at the end. Calling it again should then\n    # show two commits.\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(\"recording state in git\", out_list[1])\n\n    out, err = repo._run_annex_command('log')\n    out_list = out.rstrip(os.linesep).splitlines()\n    eq_(len(out_list), 2, \"Output:\\n%s\" % out_list)\n    assert_in(file1, out_list[0])\n    assert_in(file2, out_list[1])\n\n    # Now git knows as well:\n    out, err = runner.run(['git', 'log', 'git-annex'])\n    num_commits = len([commit\n                       for commit in out.rstrip(os.linesep).split('\\n')\n                       if commit.startswith('commit')])\n    eq_(num_commits, 4)\n\n\n@with_testrepos('basic_annex', flavors=['local'])\n@with_tempfile\ndef test_AnnexRepo_on_uninited_annex(origin, path):\n    # \"Manually\" clone to avoid initialization:\n    from datalad.cmd import Runner\n    runner = Runner()\n    _ = runner([\"git\", \"clone\", origin, path], expect_stderr=True)\n\n    assert_false(exists(opj(path, '.git', 'annex'))) # must not be there for this test to be valid\n    annex = AnnexRepo(path, create=False, init=False)  # so we can initialize without\n    # and still can get our things\n    assert_false(annex.file_has_content('test-annex.dat'))\n    with swallow_outputs():\n        annex.get('test-annex.dat')\n        ok_(annex.file_has_content('test-annex.dat'))\n\n\n@assert_cwd_unchanged\n@with_tempfile\ndef test_AnnexRepo_commit(path):\n\n    ds = AnnexRepo(path, create=True)\n    filename = opj(path, get_most_obscure_supported_name())\n    with open(filename, 'w') as f:\n        f.write(\"File to add to git\")\n    ds.add(filename, git=True)\n\n    assert_raises(AssertionError, ok_clean_git, path, annex=True)\n\n    ds.commit(\"test _commit\")\n    ok_clean_git(path, annex=True)\n\n    # nothing to commit doesn't raise by default:\n    ds.commit()\n    # but does with careless=False:\n    assert_raises(CommandError, ds.commit, careless=False)\n\n    # committing untracked file raises:\n    with open(opj(path, \"untracked\"), \"w\") as f:\n        f.write(\"some\")\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"untracked\")\n    # not existing file as well:\n    assert_raises(FileNotInRepositoryError, ds.commit, files=\"not-existing\")\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_annex(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    filename_abs = opj(repo.path, filename)\n    with open(filename_abs, \"w\") as f:\n        f.write(\"some\")\n\n    out_json = repo.add(filename)\n    # file is known to annex:\n    if not repo.is_direct_mode():\n        assert_true(os.path.islink(filename_abs),\n                    \"Annexed file is not a link.\")\n    else:\n        assert_false(os.path.islink(filename_abs),\n                     \"Annexed file is link in direct mode.\")\n    assert_in('key', out_json)\n    key = repo.get_file_key(filename)\n    assert_false(key == '')\n    assert_equal(key, out_json['key'])\n    ok_(repo.file_has_content(filename))\n\n    # uncommitted:\n    # but not in direct mode branch\n    if repo.is_direct_mode():\n        ok_(not repo.is_dirty(submodules=False))\n    else:\n        ok_(repo.is_dirty(submodules=False))\n\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename)\n    repo.commit(msg=\"Added another file to annex.\")\n    # known to annex:\n    ok_(repo.get_file_key(filename))\n    ok_(repo.file_has_content(filename))\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\ndef test_AnnexRepo_add_to_git(path):\n\n    # Note: Some test repos appears to not be initialized.\n    #       Therefore: 'init=True'\n    # TODO: Fix these repos finally!\n\n    # clone as provided by with_testrepos:\n    repo = AnnexRepo(path, create=False, init=True)\n\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n    filename = get_most_obscure_supported_name()\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"some\")\n    repo.add(filename, git=True)\n\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n    # uncommitted:\n    ok_(repo.is_dirty(submodules=False))\n    repo.commit(\"Added file to annex.\")\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n    # now using commit/msg options:\n    filename = \"another.txt\"\n    with open(opj(repo.path, filename), \"w\") as f:\n        f.write(\"something else\")\n\n    repo.add(filename, git=True)\n    repo.commit(msg=\"Added another file to annex.\")\n    # not in annex, but in git:\n    assert_raises(FileInGitError, repo.get_file_key, filename)\n\n    # and committed:\n    ok_clean_git(repo, annex=True, ignore_submodules=True)\n\n\n@with_testrepos('submodule_annex', flavors=['clone'])\ndef test_AnnexRepo_add_unexpected_direct_mode(path):\n    # tests a special case where a submodule is in direct mode, while it's\n    # superproject is not.\n    # There is no point in this test, if direct mode was enforced in the\n    # superproject already (either by test run configuration or FS) or if the\n    # repositories are in V6 by default (where there is no direct mode)\n\n    top = AnnexRepo(path)\n\n    if top.is_direct_mode() or top.config.get(\"annex.version\") == '6':\n        raise SkipTest(\"Nothing to test for\")\n\n    top.update_submodule('subm 1', init=True)\n    sub = AnnexRepo(opj(path, 'subm 1'))\n    sub.set_direct_mode(True)\n    with swallow_logs(new_level=logging.WARNING) as cml:\n        top.add('.')\n        cml.assert_logged(msg=\"Known bug in direct mode.\",\n                          level=\"WARNING\",\n                          regex=False)\n\n\n\n@ignore_nose_capturing_stdout\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_AnnexRepo_get(src, dst):\n\n    annex = AnnexRepo.clone(src, dst)\n    assert_is_instance(annex, AnnexRepo, \"AnnexRepo was not created.\")\n    testfile = 'test-annex.dat'\n    testfile_abs = opj(dst, testfile)\n    assert_false(annex.file_has_content(\"test-annex.dat\"))\n    with swallow_outputs():\n        annex.get(testfile)\n    ok_(annex.file_has_content(\"test-annex.dat\"))\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n    called = []\n    # for some reason yoh failed mock to properly just call original func\n    orig_run = annex._run_annex_command\n\n    def check_run(cmd, annex_options, **kwargs):\n        called.append(cmd)\n        if cmd == 'find':\n            assert_not_in('-J5', annex_options)\n        elif cmd == 'get':\n            assert_in('-J5', annex_options)\n        else:\n            raise AssertionError(\n                \"no other commands so far should be ran. Got %s, %s\" %\n                (cmd, annex_options)\n            )\n        return orig_run(cmd, annex_options=annex_options, **kwargs)\n\n    annex.drop(testfile)\n    with patch.object(AnnexRepo, '_run_annex_command',\n                      side_effect=check_run, auto_spec=True), \\\n            swallow_outputs():\n        annex.get(testfile, jobs=5)\n    eq_(called, ['find', 'get'])\n    ok_file_has_content(testfile_abs, \"content to be annex-addurl'd\", strip=True)\n\n\n# TODO:\n#def init_remote(self, name, options):\n#def enable_remote(self, name):\n\n@with_testrepos('basic_annex$', flavors=['clone'])\n@with_tempfile\ndef _test_AnnexRepo_get_contentlocation(batch, path, work_dir_outside):\n    annex = AnnexRepo(path, create=False, init=False)\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n    # TODO: see if we can avoid this or specify custom exception\n    eq_(annex.get_contentlocation(key, batch=batch), '')\n\n    with swallow_outputs() as cmo:\n        annex.get(fname)\n    key_location = annex.get_contentlocation(key, batch=batch)\n    assert(key_location)\n    # they both should point to the same location eventually\n    eq_(os.path.realpath(opj(annex.path, fname)),\n        os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a subdir of the annex:\n    with chpwd(opj(annex.path, 'subdir'), mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n    # test how it would look if done under a dir outside of the annex:\n    with chpwd(work_dir_outside, mkdir=True):\n        key_location = annex.get_contentlocation(key, batch=batch)\n        # they both should point to the same location eventually\n        eq_(os.path.realpath(opj(annex.path, fname)),\n            os.path.realpath(opj(annex.path, key_location)))\n\n\ndef test_AnnexRepo_get_contentlocation():\n    for batch in (False, True):\n        yield _test_AnnexRepo_get_contentlocation, batch\n\n\n@with_tree(tree=(('about.txt', 'Lots of abouts'),\n                 ('about2.txt', 'more abouts'),\n                 ('about2_.txt', 'more abouts_'),\n                 ('d', {'sub.txt': 'more stuff'})))\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_to_file_batched(sitepath, siteurl, dst):\n\n    if os.environ.get('DATALAD_FAKE__DATES'):\n        raise SkipTest(\n            \"Faked dates are enabled; skipping batched addurl tests\")\n\n    ar = AnnexRepo(dst, create=True)\n    testurl = urljoin(siteurl, 'about.txt')\n    testurl2 = urljoin(siteurl, 'about2.txt')\n    testurl2_ = urljoin(siteurl, 'about2_.txt')\n    testurl3 = urljoin(siteurl, 'd/sub.txt')\n    url_file_prefix = urlsplit(testurl).netloc.split(':')[0]\n    testfile = 'about.txt'\n    testfile2 = 'about2.txt'\n    testfile2_ = 'about2_.txt'\n    testfile3 = opj('d', 'sub.txt')\n\n    # add to an existing but not committed file\n    # TODO: __call__ of the BatchedAnnex must be checked to be called\n    copyfile(opj(sitepath, 'about.txt'), opj(dst, testfile))\n    # must crash sensibly since file exists, we shouldn't addurl to non-annexed files\n    with assert_raises(AnnexBatchCommandError):\n        ar.add_url_to_file(testfile, testurl, batch=True)\n\n    # Remove it and re-add\n    unlink(opj(dst, testfile))\n    ar.add_url_to_file(testfile, testurl, batch=True)\n\n    info = ar.info(testfile)\n    eq_(info['size'], 14)\n    assert(info['key'])\n    # not even added to index yet since we this repo is with default batch_size\n    # but: in direct mode it is added!\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n\n    # TODO: none of the below should re-initiate the batch process\n\n    # add to an existing and staged annex file\n    copyfile(opj(sitepath, 'about2.txt'), opj(dst, testfile2))\n    ar.add(testfile2)\n    ar.add_url_to_file(testfile2, testurl2, batch=True)\n    assert(ar.info(testfile2))\n    # not committed yet\n    # assert_in(ar.WEB_UUID, ar.whereis(testfile2))\n\n    # add to an existing and committed annex file\n    copyfile(opj(sitepath, 'about2_.txt'), opj(dst, testfile2_))\n    ar.add(testfile2_)\n    if ar.is_direct_mode():\n        assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    else:\n        assert_not_in(ar.WEB_UUID, ar.whereis(testfile))\n    ar.commit(\"added about2_.txt and there was about2.txt lingering around\")\n    # commit causes closing all batched annexes, so testfile gets committed\n    assert_in(ar.WEB_UUID, ar.whereis(testfile))\n    assert(not ar.dirty)\n    ar.add_url_to_file(testfile2_, testurl2_, batch=True)\n    assert(ar.info(testfile2_))\n    assert_in(ar.WEB_UUID, ar.whereis(testfile2_))\n\n    # add into a new file\n    # filename = 'newfile.dat'\n    filename = get_most_obscure_supported_name()\n\n    # Note: The following line was necessary, since the test setup just\n    # doesn't work with singletons\n    # TODO: Singleton mechanic needs a general solution for this\n    AnnexRepo._unique_instances.clear()\n    ar2 = AnnexRepo(dst, batch_size=1)\n\n    with swallow_outputs():\n        eq_(len(ar2._batched), 0)\n        ar2.add_url_to_file(filename, testurl, batch=True)\n        eq_(len(ar2._batched), 1)  # we added one more with batch_size=1\n        ar2.precommit()  # to possibly stop batch process occupying the stdout\n    ar2.commit(\"added new file\")  # would do nothing ATM, but also doesn't fail\n    assert_in(filename, ar2.get_files())\n    assert_in(ar.WEB_UUID, ar2.whereis(filename))\n\n    if not ar.is_direct_mode():\n        # in direct mode there's nothing to commit\n        ar.commit(\"actually committing new files\")\n    assert_in(filename, ar.get_files())\n    assert_in(ar.WEB_UUID, ar.whereis(filename))\n    # this poor bugger still wasn't added since we used default batch_size=0 on him\n\n    # and closing the pipes now shoudn't anyhow affect things\n    eq_(len(ar._batched), 1)\n    ar._batched.close()\n    eq_(len(ar._batched), 1)  # doesn't remove them, just closes\n    assert(not ar.dirty)\n\n    ar._batched.clear()\n    eq_(len(ar._batched), 0)  # .clear also removes\n\n    raise SkipTest(\"TODO: more, e.g. add with a custom backend\")\n    # TODO: also with different modes (relaxed, fast)\n    # TODO: verify that file is added with that backend and that we got a new batched process\n\n\n@with_tree(tree={\"foo\": \"foo content\"})\n@serve_path_via_http()\n@with_tree(tree={\"bar\": \"bar content\"})\ndef test_annexrepo_fake_dates_disables_batched(sitepath, siteurl, dst):\n    ar = AnnexRepo(dst, create=True, fake_dates=True)\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.add_url_to_file(\"foo-dst\", urljoin(siteurl, \"foo\"), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching addurl call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n    ar.add(\"bar\")\n    ar.commit(\"add bar\")\n\n    with swallow_logs(new_level=logging.DEBUG) as cml:\n        ar.drop_key(ar.get_file_key([\"bar\"]), batch=True)\n        cml.assert_logged(\n            msg=\"Not batching drop_key call because fake dates are enabled\",\n            level=\"DEBUG\",\n            regex=False)\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_backends(path):\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, None)\n\n    rmtree(path)\n\n    repo = AnnexRepo(path, backend='MD5E')\n    eq_(repo.default_backends, ['MD5E'])\n\n    # persists\n    repo = AnnexRepo(path)\n    eq_(repo.default_backends, ['MD5E'])\n\n\n@skip_ssh\n@with_tempfile\n@with_testrepos('basic_annex', flavors=['local'])\n@with_testrepos('basic_annex', flavors=['local'])\ndef test_annex_ssh(repo_path, remote_1_path, remote_2_path):\n    from datalad import ssh_manager\n    # create remotes:\n    rm1 = AnnexRepo(remote_1_path, create=False)\n    rm2 = AnnexRepo(remote_2_path, create=False)\n\n    # check whether we are the first to use these sockets:\n    socket_1 = opj(ssh_manager.socket_dir, get_connection_hash('datalad-test'))\n    socket_2 = opj(ssh_manager.socket_dir, get_connection_hash('localhost'))\n    datalad_test_was_open = exists(socket_1)\n    localhost_was_open = exists(socket_2)\n\n    # repo to test:AnnexRepo(repo_path)\n    # At first, directly use git to add the remote, which should be recognized\n    # by AnnexRepo's constructor\n    gr = GitRepo(repo_path, create=True)\n    AnnexRepo(repo_path)\n    gr.add_remote(\"ssh-remote-1\", \"ssh://datalad-test\" + remote_1_path)\n\n    # Now, make it an annex:\n    ar = AnnexRepo(repo_path, create=False)\n\n    # connection to 'datalad-test' should be known to ssh manager:\n    assert_in(socket_1, ssh_manager._connections)\n    # but socket was not touched:\n    if datalad_test_was_open:\n        ok_(exists(socket_1))\n    else:\n        ok_(not exists(socket_1))\n\n    from datalad import lgr\n    # remote interaction causes socket to be created:\n    try:\n        # Note: For some reason, it hangs if log_stdout/err True\n        # TODO: Figure out what's going on\n        #  yoh: I think it is because of what is \"TODOed\" within cmd.py --\n        #       trying to log/obtain both through PIPE could lead to lock\n        #       downs.\n        # here we use our swallow_logs to overcome a problem of running under\n        # nosetests without -s, when nose then tries to swallow stdout by\n        # mocking it with StringIO, which is not fully compatible with Popen\n        # which needs its .fileno()\n        with swallow_outputs():\n            ar._run_annex_command('sync',\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_1))\n\n    # add another remote:\n    ar.add_remote('ssh-remote-2', \"ssh://localhost\" + remote_2_path)\n\n    # now, this connection to localhost was requested:\n    assert_in(socket_2, ssh_manager._connections)\n    # but socket was not touched:\n    if localhost_was_open:\n        # FIXME: occasionally(?) fails in V6:\n        if not ar.config.getint(\"annex\", \"version\") == 6:\n            ok_(exists(socket_2))\n    else:\n        ok_(not exists(socket_2))\n\n    # sync with the new remote:\n    try:\n        with swallow_outputs():\n            ar._run_annex_command('sync', annex_options=['ssh-remote-2'],\n                                  expect_stderr=True,\n                                  log_stdout=False,\n                                  log_stderr=False,\n                                  expect_fail=True)\n    # sync should return exit code 1, since it can not merge\n    # doesn't matter for the purpose of this test\n    except CommandError as e:\n        if e.code == 1:\n            pass\n\n    ok_(exists(socket_2))\n    ssh_manager.close(ctrl_path=[socket_1, socket_2])\n\n@with_testrepos('basic_annex', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_remove(path1, path2):\n    ar1 = AnnexRepo(path1, create=False)\n    ar2 = AnnexRepo.clone(path1, path2, create=True, direct=True)\n\n    for repo in (ar1, ar2):\n        file_list = repo.get_annexed_files()\n        assert len(file_list) >= 1\n        # remove a single file\n        out = repo.remove(file_list[0])\n        assert_not_in(file_list[0], repo.get_annexed_files())\n        eq_(out[0], file_list[0])\n\n        with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n            f.write(\"whatever\")\n\n        # add it\n        repo.add(\"rm-test.dat\")\n\n        # remove without '--force' should fail, due to staged changes:\n        if repo.is_direct_mode():\n            assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n        else:\n            assert_raises(GitCommandError, repo.remove, \"rm-test.dat\")\n        assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n        # now force:\n        out = repo.remove(\"rm-test.dat\", force=True)\n        assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n        eq_(out[0], \"rm-test.dat\")\n\n\n@with_tempfile\n@with_tempfile\n@with_tempfile\ndef test_repo_version(path1, path2, path3):\n    annex = AnnexRepo(path1, create=True, version=6)\n    ok_clean_git(path1, annex=True)\n    version = annex.repo.config_reader().get_value('annex', 'version')\n    eq_(version, 6)\n\n    # default from config item (via env var):\n    with patch.dict('os.environ', {'DATALAD_REPO_VERSION': '6'}):\n        annex = AnnexRepo(path2, create=True)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 6)\n\n        # parameter `version` still has priority over default config:\n        annex = AnnexRepo(path3, create=True, version=5)\n        version = annex.repo.config_reader().get_value('annex', 'version')\n        eq_(version, 5)\n\n\n@with_testrepos('.*annex.*', flavors=['clone'])\n@with_tempfile(mkdir=True)\ndef test_annex_copy_to(origin, clone):\n    repo = AnnexRepo(origin, create=False)\n    remote = AnnexRepo.clone(origin, clone, create=True)\n    repo.add_remote(\"target\", clone)\n\n    assert_raises(IOError, repo.copy_to, \"doesnt_exist.dat\", \"target\")\n    assert_raises(FileInGitError, repo.copy_to, \"INFO.txt\", \"target\")\n    assert_raises(ValueError, repo.copy_to, \"test-annex.dat\", \"invalid_target\")\n\n    # test-annex.dat has no content to copy yet:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [])\n\n    repo.get(\"test-annex.dat\")\n    # now it has:\n    eq_(repo.copy_to(\"test-annex.dat\", \"target\"), [\"test-annex.dat\"])\n    # and will not be copied again since it was already copied\n    eq_(repo.copy_to([\"INFO.txt\", \"test-annex.dat\"], \"target\"), [])\n\n    # Test that if we pass a list of items and annex processes them nicely,\n    # we would obtain a list back. To not stress our tests even more -- let's mock\n    def ok_copy(command, **kwargs):\n        # Check that we do pass to annex call only the list of files which we\n        #  asked to be copied\n        assert_in('copied1', kwargs['annex_options'])\n        assert_in('copied2', kwargs['annex_options'])\n        assert_in('existed', kwargs['annex_options'])\n        return \"\"\"\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied1\"}\n{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"copied2\"}\n{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey3\", \"file\":\"existed\"}\n\"\"\", \"\"\n    with patch.object(repo, '_run_annex_command', ok_copy):\n        eq_(repo.copy_to([\"copied2\", \"copied1\", \"existed\"], \"target\"),\n            [\"copied1\", \"copied2\"])\n\n    # now let's test that we are correctly raising the exception in case if\n    # git-annex execution fails\n    orig_run = repo._run_annex_command\n\n    # Kinda a bit off the reality since no nonex* would not be returned/handled\n    # by _get_expected_files, so in real life -- wouldn't get report about Incomplete!?\n    def fail_to_copy(command, **kwargs):\n        if command == 'copy':\n            # That is not how annex behaves\n            # http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            # for non-existing files output goes into stderr\n            raise CommandError(\n                \"Failed to run ...\",\n                stdout=\n                    '{\"command\":\"copy\",\"note\":\"to target ...\", \"success\":true, \"key\":\"akey1\", \"file\":\"copied\"}\\n'\n                    '{\"command\":\"copy\",\"note\":\"checking target ...\", \"success\":true, \"key\":\"akey2\", \"file\":\"existed\"}\\n',\n                stderr=\n                    'git-annex: nonex1 not found\\n'\n                    'git-annex: nonex2 not found\\n'\n            )\n        else:\n            return orig_run(command, **kwargs)\n\n    def fail_to_copy_get_expected(files, expr):\n        assert files == [\"copied\", \"existed\", \"nonex1\", \"nonex2\"]\n        return {'akey1': 10}, [\"copied\"]\n\n    with patch.object(repo, '_run_annex_command', fail_to_copy), \\\n            patch.object(repo, '_get_expected_files', fail_to_copy_get_expected):\n        with assert_raises(IncompleteResultsError) as cme:\n            repo.copy_to([\"copied\", \"existed\", \"nonex1\", \"nonex2\"], \"target\")\n    eq_(cme.exception.results, [\"copied\"])\n    eq_(cme.exception.failed, ['nonex1', 'nonex2'])\n\n\n\n@with_testrepos('.*annex.*', flavors=['local'])\n# TODO: flavor 'network' has wrong content for test-annex.dat!\n@with_tempfile\ndef test_annex_drop(src, dst):\n    ar = AnnexRepo.clone(src, dst)\n    testfile = 'test-annex.dat'\n    assert_false(ar.file_has_content(testfile))\n    ar.get(testfile)\n    ok_(ar.file_has_content(testfile))\n\n    # drop file by name:\n    result = ar.drop([testfile])\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['file'], testfile)\n\n    ar.get(testfile)\n\n    # drop file by key:\n    testkey = ar.get_file_key(testfile)\n    result = ar.drop([testkey], key=True)\n    assert_false(ar.file_has_content(testfile))\n    ok_(isinstance(result, list))\n    eq_(len(result), 1)\n    eq_(result[0]['command'], 'drop')\n    eq_(result[0]['success'], True)\n    eq_(result[0]['key'], testkey)\n\n    # insufficient arguments:\n    assert_raises(TypeError, ar.drop)\n    assert_raises(InsufficientArgumentsError, ar.drop, [], options=[\"--jobs=5\"])\n    assert_raises(InsufficientArgumentsError, ar.drop, [])\n\n    # too much arguments:\n    assert_raises(CommandError, ar.drop, ['.'], options=['--all'])\n\n\n@with_tree({\"a.txt\": \"a\", \"b.txt\": \"b\", \"c.py\": \"c\", \"d\": \"d\"})\ndef test_annex_get_annexed_files(path):\n    repo = AnnexRepo(path)\n    repo.add(\".\")\n    repo.commit()\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n\n    repo.drop(\"a.txt\", options=[\"--force\"])\n    eq_(set(repo.get_annexed_files()), {\"a.txt\", \"b.txt\", \"c.py\", \"d\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        {\"b.txt\", \"c.py\", \"d\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\"])),\n        {\"a.txt\", \"b.txt\"})\n    eq_(set(repo.get_annexed_files(with_content_only=True,\n                                   patterns=[\"*.txt\"])),\n        {\"b.txt\"})\n\n    eq_(set(repo.get_annexed_files(patterns=[\"*.txt\", \"*.py\"])),\n        {\"a.txt\", \"b.txt\", \"c.py\"})\n\n    eq_(set(repo.get_annexed_files()),\n        set(repo.get_annexed_files(patterns=[\"*\"])))\n\n    eq_(set(repo.get_annexed_files(with_content_only=True)),\n        set(repo.get_annexed_files(with_content_only=True, patterns=[\"*\"])))\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_annex_remove(path):\n    repo = AnnexRepo(path, create=False)\n\n    file_list = repo.get_annexed_files()\n    assert len(file_list) >= 1\n    # remove a single file\n    out = repo.remove(file_list[0])\n    assert_not_in(file_list[0], repo.get_annexed_files())\n    eq_(out[0], file_list[0])\n\n    with open(opj(repo.path, \"rm-test.dat\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    # add it\n    repo.add(\"rm-test.dat\")\n\n    # remove without '--force' should fail, due to staged changes:\n    assert_raises(CommandError, repo.remove, \"rm-test.dat\")\n    assert_in(\"rm-test.dat\", repo.get_annexed_files())\n\n    # now force:\n    out = repo.remove(\"rm-test.dat\", force=True)\n    assert_not_in(\"rm-test.dat\", repo.get_annexed_files())\n    eq_(out[0], \"rm-test.dat\")\n\n\n@with_batch_direct\n@with_testrepos('basic_annex', flavors=['clone'], count=1)\ndef test_is_available(batch, direct, p):\n    annex = AnnexRepo(p)\n\n    # bkw = {'batch': batch}\n    if batch:\n        is_available = partial(annex.is_available, batch=batch)\n    else:\n        is_available = annex.is_available\n\n    fname = 'test-annex.dat'\n    key = annex.get_file_key(fname)\n\n    # explicit is to verify data type etc\n    assert is_available(key, key=True) is True\n    assert is_available(fname) is True\n\n    # known remote but doesn't have it\n    assert is_available(fname, remote='origin') is False\n    # it is on the 'web'\n    assert is_available(fname, remote='web') is True\n    # not effective somehow :-/  may be the process already running or smth\n    # with swallow_logs(), swallow_outputs():  # it will complain!\n    assert is_available(fname, remote='unknown') is False\n    assert_false(is_available(\"boguskey\", key=True))\n\n    # remove url\n    urls = annex.get_urls(fname) #, **bkw)\n    assert(len(urls) == 1)\n    annex.rm_url(fname, urls[0])\n\n    assert is_available(key, key=True) is False\n    assert is_available(fname) is False\n    assert is_available(fname, remote='web') is False\n\n\n@with_tempfile(mkdir=True)\ndef test_annex_add_no_dotfiles(path):\n    ar = AnnexRepo(path, create=True)\n    print(ar.path)\n    assert_true(os.path.exists(ar.path))\n    assert_false(ar.dirty)\n    os.makedirs(opj(ar.path, '.datalad'))\n    # we don't care about empty directories\n    assert_false(ar.dirty)\n    with open(opj(ar.path, '.datalad', 'somefile'), 'w') as f:\n        f.write('some content')\n    # make sure the repo is considered dirty now\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # no file is being added, as dotfiles/directories are ignored by default\n    ar.add('.', git=False)\n    # double check, still dirty\n    assert_true(ar.dirty)  # TODO: has been more detailed assertion (untracked file)\n    # now add to git, and it should work\n    ar.add('.', git=True)\n    # all in index\n    assert_true(ar.dirty)\n    # TODO: has been more specific:\n    # assert_false(ar.repo.is_dirty(\n    #     index=False, working_tree=True, untracked_files=True, submodules=True))\n    ar.commit(msg=\"some\")\n    # all committed\n    assert_false(ar.dirty)\n    # not known to annex\n    assert_false(ar.is_under_annex(opj(ar.path, '.datalad', 'somefile')))\n\n\n@with_tempfile\ndef test_annex_version_handling(path):\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n         patch.object(AnnexRepo, '_check_git_annex_version',\n                      auto_spec=True,\n                      side_effect=AnnexRepo._check_git_annex_version) \\\n            as cmpc, \\\n         patch.object(external_versions, '_versions',\n                      {'cmd:annex': AnnexRepo.GIT_ANNEX_MIN_VERSION}):\n            eq_(AnnexRepo.git_annex_version, None)\n            ar1 = AnnexRepo(path, create=True)\n            assert(ar1)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n            # 2nd time must not be called\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            ar2 = AnnexRepo(path)\n            assert(ar2)\n            eq_(AnnexRepo.git_annex_version, AnnexRepo.GIT_ANNEX_MIN_VERSION)\n            eq_(cmpc.call_count, 1)\n    with patch.object(AnnexRepo, 'git_annex_version', None) as cmpov, \\\n            patch.object(AnnexRepo, '_check_git_annex_version',\n                         auto_spec=True,\n                         side_effect=AnnexRepo._check_git_annex_version):\n        # no git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': None}):\n            eq_(AnnexRepo.git_annex_version, None)\n            with assert_raises(MissingExternalDependency) as cme:\n                try:\n                    # Note: Remove to cause creation of a new instance\n                    rmtree(path)\n                except OSError:\n                    pass\n                AnnexRepo(path)\n            if linux_distribution_name == 'debian':\n                assert_in(\"http://neuro.debian.net\", str(cme.exception))\n            eq_(AnnexRepo.git_annex_version, None)\n\n        # outdated git-annex at all\n        with patch.object(\n                external_versions, '_versions', {'cmd:annex': '6.20160505'}):\n            eq_(AnnexRepo.git_annex_version, None)\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n            # and we don't assign it\n            eq_(AnnexRepo.git_annex_version, None)\n            # so we could still fail\n            try:\n                # Note: Remove to cause creation of a new instance\n                rmtree(path)\n            except OSError:\n                pass\n            assert_raises(OutdatedExternalDependency, AnnexRepo, path)\n\n\ndef test_ProcessAnnexProgressIndicators():\n    irrelevant_lines = (\n        'abra',\n        '{\"some_json\": \"sure thing\"}'\n    )\n    # regular lines, without completion for known downloads\n    success_lines = (\n        '{\"command\":\"get\",\"note\":\"\",\"success\":true,\"key\":\"key1\",\"file\":\"file1\"}',\n        '{\"command\":\"comm\",\"note\":\"\",\"success\":true,\"key\":\"backend-s10--key2\"}',\n    )\n    progress_lines = (\n        '{\"byte-progress\":10,\"action\":{\"command\":\"get\",\"note\":\"from web...\",'\n            '\"key\":\"key1\",\"file\":\"file1\"},\"percent-progress\":\"10%\"}',\n    )\n\n    # without providing expected entries\n    proc = ProcessAnnexProgressIndicators()\n    # when without any target downloads, there is no total_pbar\n    assert_is(proc.total_pbar, None)\n    # for regular lines -- should just return them without side-effects\n    for l in irrelevant_lines + success_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    eq_(proc(progress_lines[0]), None)\n    eq_(len(proc.pbars), 1)\n    # but when we finish download -- should get cleared\n    eq_(proc(success_lines[0]), success_lines[0])\n    eq_(proc.pbars, {})\n    # and no side-effect of any kind in finish\n    eq_(proc.finish(), None)\n\n    proc = ProcessAnnexProgressIndicators(expected={'key1': 100, 'key2': None})\n    # when without any target downloads, there is no total_pbar\n    assert(proc.total_pbar is not None)\n    eq_(proc.total_pbar.total, 100)  # as much as it knows at this point\n    eq_(proc.total_pbar.current, 0)\n    # for regular lines -- should still just return them without side-effects\n    for l in irrelevant_lines:\n        with swallow_outputs() as cmo:\n            eq_(proc(l), l)\n            eq_(proc.pbars, {})\n            eq_(cmo.out, '')\n            eq_(cmo.err, '')\n    # should process progress lines\n    # it doesn't swallow everything -- so there will be side-effects in output\n    with swallow_outputs() as cmo:\n        eq_(proc(progress_lines[0]), None)\n        eq_(len(proc.pbars), 1)\n        # but when we finish download -- should get cleared\n        eq_(proc(success_lines[0]), success_lines[0])\n        eq_(proc.pbars, {})\n        out = cmo.out\n\n    from datalad.ui import ui\n    from datalad.ui.dialog import SilentConsoleLog\n\n    assert out \\\n        if not isinstance(ui.ui, SilentConsoleLog) else not out\n    assert proc.total_pbar is not None\n    # and no side-effect of any kind in finish\n    with swallow_outputs() as cmo:\n        eq_(proc.finish(), None)\n        eq_(proc.total_pbar, None)\n\n\n@with_tempfile\n@with_tempfile\ndef test_get_description(path1, path2):\n    annex1 = AnnexRepo(path1, create=True)\n    # some content for git-annex branch\n    create_tree(path1, {'1.dat': 'content'})\n    annex1.add('1.dat', git=False)\n    annex1.commit(\"msg\")\n    annex1_description = annex1.get_description()\n    assert_not_equal(annex1_description, path1)\n\n    annex2 = AnnexRepo(path2, create=True, description='custom 2')\n    eq_(annex2.get_description(), 'custom 2')\n    # not yet known\n    eq_(annex2.get_description(uuid=annex1.uuid), None)\n\n    annex2.add_remote('annex1', path1)\n    annex2.fetch('annex1')\n    # it will match the remote name\n    eq_(annex2.get_description(uuid=annex1.uuid),\n        annex1_description + ' [annex1]')\n    # add a little probe file to make sure it stays untracked\n    create_tree(path1, {'probe': 'probe'})\n    assert_not_in('probe', annex2.get_indexed_files())\n    annex2.merge_annex('annex1')\n    assert_not_in('probe', annex2.get_indexed_files())\n    # but let's remove the remote\n    annex2.remove_remote('annex1')\n    eq_(annex2.get_description(uuid=annex1.uuid), annex1_description)\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_flyweight(path1, path2):\n\n    repo1 = AnnexRepo(path1, create=True)\n    assert_is_instance(repo1, AnnexRepo)\n    # instantiate again:\n    repo2 = AnnexRepo(path1, create=False)\n    assert_is_instance(repo2, AnnexRepo)\n    # the very same object:\n    ok_(repo1 is repo2)\n\n    # reference the same in an different way:\n    with chpwd(path1):\n        repo3 = AnnexRepo(relpath(path1, start=path2), create=False)\n        assert_is_instance(repo3, AnnexRepo)\n    # it's the same object:\n    ok_(repo1 is repo3)\n\n    # but path attribute is absolute, so they are still equal:\n    ok_(repo1 == repo3)\n\n    # Now, let's try to get a GitRepo instance from a path, we already have an\n    # AnnexRepo of\n    repo4 = GitRepo(path1)\n    assert_is_instance(repo4, GitRepo)\n    assert_not_is_instance(repo4, AnnexRepo)\n\n\n@with_testrepos(flavors=local_testrepo_flavors)\n@with_tempfile(mkdir=True)\n@with_tempfile\ndef test_AnnexRepo_get_toppath(repo, tempdir, repo2):\n\n    reporeal = realpath(repo)\n    eq_(AnnexRepo.get_toppath(repo, follow_up=False), reporeal)\n    eq_(AnnexRepo.get_toppath(repo), repo)\n    # Generate some nested directory\n    AnnexRepo(repo2, create=True)\n    repo2real = realpath(repo2)\n    nested = opj(repo2, \"d1\", \"d2\")\n    os.makedirs(nested)\n    eq_(AnnexRepo.get_toppath(nested, follow_up=False), repo2real)\n    eq_(AnnexRepo.get_toppath(nested), repo2)\n    # and if not under git, should return None\n    eq_(AnnexRepo.get_toppath(tempdir), None)\n\n\n@with_testrepos(\".*basic.*\", flavors=['local'])\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_add_submodule(source, path):\n\n    top_repo = AnnexRepo(path, create=True)\n\n    top_repo.add_submodule('sub', name='sub', url=source)\n    top_repo.commit('submodule added')\n    eq_([s.name for s in top_repo.get_submodules()], ['sub'])\n\n    ok_clean_git(top_repo, annex=True)\n    ok_clean_git(opj(path, 'sub'), annex=False)\n\n\ndef test_AnnexRepo_update_submodule():\n    raise SkipTest(\"TODO\")\n\n\n@known_failure_v6  #FIXME\ndef test_AnnexRepo_get_submodules():\n    raise SkipTest(\"TODO\")\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_dirty(path):\n\n    repo = AnnexRepo(path, create=True)\n    ok_(not repo.dirty)\n\n    # pure git operations:\n    # untracked file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    ok_(repo.dirty)\n    # staged file\n    repo.add('file1.txt', git=True)\n    ok_(repo.dirty)\n    # clean again\n    repo.commit(\"file1.txt added\")\n    ok_(not repo.dirty)\n    # modify to be the same\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('whatever')\n    if not repo.config.getint(\"annex\", \"version\") == 6:\n        ok_(not repo.dirty)\n    # modified file\n    with open(opj(path, 'file1.txt'), 'w') as f:\n        f.write('something else')\n    ok_(repo.dirty)\n    # clean again\n    repo.add('file1.txt', git=True)\n    repo.commit(\"file1.txt modified\")\n    ok_(not repo.dirty)\n\n    # annex operations:\n    # untracked file\n    with open(opj(path, 'file2.txt'), 'w') as f:\n        f.write('different content')\n    ok_(repo.dirty)\n    # annexed file\n    repo.add('file2.txt', git=False)\n    if not repo.is_direct_mode():\n        # in direct mode 'annex add' results in a clean repo\n        ok_(repo.dirty)\n        # commit\n        repo.commit(\"file2.txt annexed\")\n    ok_(not repo.dirty)\n\n    # TODO: unlock/modify\n\n    # TODO: submodules\n\n\ndef _test_status(ar):\n    # TODO: plain git submodule and even deeper hierarchy?\n    #       => complete recursion to work if started from within plain git;\n    #       But this is then relevant for Dataset.status() - not herein\n\n    def sync_wrapper(push=False, pull=False, commit=False):\n        # wraps common annex-sync call, since it currently fails under\n        # mysterious circumstances in V6 adjusted branch setups\n        try:\n            ar.sync(push=push, pull=pull, commit=commit)\n        except CommandError as e:\n            if \"fatal: entry 'submod' object type (blob) doesn't match mode type \" \\\n               \"(commit)\" in e.stderr:\n                # some bug in adjusted branch(?) + submodule\n                # TODO: figure out and probably report\n                # stdout:\n                # commit\n                # [adjusted/master(unlocked) ae3e9a7] git-annex in ben@tree:/tmp/datalad_temp_test_AnnexRepo_status7qKhRQ\n                #  4 files changed, 4 insertions(+), 2 deletions(-)\n                #  create mode 100644 fifth\n                #  create mode 100644 sub/third\n                # ok\n                #\n                # failed\n                # stderr:\n                # fatal: entry 'submod' object type (blob) doesn't match mode type (commit)\n                # git-annex: user error (git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"mktree\",\"--batch\",\"-z\"] exited 128)\n                # git-annex: sync: 1 failed\n\n                # But it almost works - so apperently nothing to do\n                import logging\n                lgr = logging.getLogger(\"datalad.support.tests.test-status\")\n                lgr.warning(\"DEBUG: v6 sync failure\")\n\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n\n    # untracked files:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"it's huge!\")\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"looser\")\n\n    stat['untracked'].append('first')\n    stat['untracked'].append('second')\n    eq_(stat, ar.get_status())\n\n    # add a file to git\n    ar.add('first', git=True)\n    sync_wrapper()\n    stat['untracked'].remove('first')\n    stat['added'].append('first')\n    eq_(stat, ar.get_status())\n\n    # add a file to annex\n    ar.add('second')\n    sync_wrapper()\n    stat['untracked'].remove('second')\n    if not ar.is_direct_mode():\n        # in direct mode annex-status doesn't report an added file 'added'\n        stat['added'].append('second')\n    eq_(stat, ar.get_status())\n\n    # commit to be clean again:\n    ar.commit(\"added first and second\")\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n    eq_(stat, ar.get_status())\n    # create a file to be unannexed:\n    with open(opj(ar.path, 'fifth'), 'w') as f:\n        f.write(\"total disaster\")\n\n    ar.add('fifth')\n    sync_wrapper()\n    # TODO:\n    # Note: For some reason this seems to be the only place, where we actually\n    # need to call commit via annex-proxy. If called via '-c core.bare=False'\n    # and/or '--work-tree=.' the file ends up in git instead of annex.\n    # Note 2: This is only if we explicitly pass a path. Otherwise it works\n    # without annex-proxy.\n    ar.commit(msg=\"fifth to be unannexed\", files='fifth',\n              proxy=ar.is_direct_mode())\n    eq_(stat, ar.get_status())\n\n    ar.unannex('fifth')\n\n    sync_wrapper(pull=False, push=False, commit=True)\n    stat['untracked'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # modify a file in git:\n    with open(opj(ar.path, 'first'), 'w') as f:\n        f.write(\"increased tremendousness\")\n    stat['modified'].append('first')\n    eq_(stat, ar.get_status())\n\n    # modify an annexed file:\n    if not ar.is_direct_mode():\n        # actually: if 'second' isn't locked, which is the case in direct mode\n        ar.unlock('second')\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].append('second')\n        eq_(stat, ar.get_status())\n    with open(opj(ar.path, 'second'), 'w') as f:\n        f.write(\"Needed to unlock first. Sad!\")\n    if not ar.is_direct_mode():\n        ar.add('second')  # => modified\n        if not ar.get_active_branch().endswith('(unlocked)'):\n            stat['type_changed'].remove('second')\n    stat['modified'].append('second')\n    sync_wrapper()\n    eq_(stat, ar.get_status())\n\n    # create something in a subdir\n    os.mkdir(opj(ar.path, 'sub'))\n    with open(opj(ar.path, 'sub', 'third'), 'w') as f:\n        f.write(\"tired of winning\")\n\n    # Note, that this is different from 'git status',\n    # which would just say 'sub/':\n    stat['untracked'].append(opj('sub', 'third'))\n    eq_(stat, ar.get_status())\n\n    # test parameters for status to restrict results:\n    # limit requested states:\n    limited_status = ar.get_status(untracked=True, deleted=False, modified=True,\n                                   added=True, type_changed=False)\n    eq_(len(limited_status), 3)\n    ok_(all([k in ('untracked', 'modified', 'added') for k in limited_status]))\n    eq_(stat['untracked'], limited_status['untracked'])\n    eq_(stat['modified'], limited_status['modified'])\n    eq_(stat['added'], limited_status['added'])\n    # limit requested files:\n    limited_status = ar.get_status(path=opj('sub', 'third'))\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    ok_(all([len(limited_status[l]) == 0 for l in ('modified', 'added',\n                                                   'deleted', 'type_changed')]))\n    # again, with a list:\n    limited_status = ar.get_status(path=[opj('sub', 'third'), 'second'])\n    eq_(limited_status['untracked'], [opj('sub', 'third')])\n    eq_(limited_status['modified'], ['second'])\n    ok_(all([len(limited_status[l]) == 0 for l in ('added', 'deleted',\n                                                   'type_changed')]))\n\n    # create a subrepo:\n    sub = AnnexRepo(opj(ar.path, 'submod'), create=True)\n    # nothing changed, it's empty besides .git, which is ignored\n    eq_(stat, ar.get_status())\n\n    # file in subrepo\n    with open(opj(ar.path, 'submod', 'fourth'), 'w') as f:\n        f.write(\"this is a birth certificate\")\n    stat['untracked'].append(opj('submod', 'fourth'))\n    eq_(stat, ar.get_status())\n\n    # add to subrepo\n    sub.add('fourth')\n    sub.commit(msg=\"birther mod init'ed\")\n    stat['untracked'].remove(opj('submod', 'fourth'))\n\n    if ar.get_active_branch().endswith('(unlocked)') and \\\n       'adjusted' in ar.get_active_branch():\n        # we are running on adjusted branch => do it in submodule, too\n        sub.adjust()\n\n    # Note, that now the non-empty repo is untracked\n    stat['untracked'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the submodule\n    ar.add_submodule('submod', url=opj(curdir, 'submod'))\n\n    stat['untracked'].remove('submod/')\n    stat['added'].append('.gitmodules')\n\n    # 'submod/' might either be reported as 'added' or 'modified'.\n    # Therefore more complex assertions at this point:\n    reported_stat = ar.get_status()\n    eq_(stat['untracked'], reported_stat['untracked'])\n    eq_(stat['deleted'], reported_stat['deleted'])\n    eq_(stat['type_changed'], reported_stat['type_changed'])\n    ok_(stat['added'] == reported_stat['added'] or\n        stat['added'] + ['submod/'] == reported_stat['added'])\n    ok_(stat['modified'] == reported_stat['modified'] or\n        stat['modified'] + ['submod/'] == reported_stat['modified'])\n\n    # simpler assertion if we ignore submodules:\n    eq_(stat, ar.get_status(submodules=False))\n\n    # commit the submodule\n    # in direct mode, commit of a removed submodule fails with:\n    #  error: unable to index file submod\n    #  fatal: updating files failed\n    #\n    # - this happens, when commit is called with -c core.bare=False\n    # - it works when called via annex proxy\n    # - if we add a submodule instead of removing one, it's vice versa with\n    #   the very same error message\n\n    ar.commit(msg=\"submodule added\", files=['.gitmodules', 'submod'])\n\n    stat['added'].remove('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # add another file to submodule\n    with open(opj(ar.path, 'submod', 'not_tracked'), 'w') as f:\n        f.write(\"#LastNightInSweden\")\n    stat['modified'].append('submod/')\n    eq_(stat, ar.get_status())\n\n    # add the untracked file:\n    sub.add('not_tracked')\n    if sub.is_direct_mode():\n        # 'sub' is now considered to be clean; therefore it's not reported as\n        # modified upwards\n        # This is consistent in a way, but surprising in another ...\n        pass\n    else:\n        eq_(stat, ar.get_status())\n    sub.commit(msg=\"added file not_tracked\")\n    # 'submod/' still modified when looked at from above:\n    eq_(stat, ar.get_status())\n\n    ar.add('submod', git=True)\n    ar.commit(msg=\"submodule modified\", files='submod')\n    stat['modified'].remove('submod/')\n    eq_(stat, ar.get_status())\n\n    # clean again:\n    stat = {'untracked': [],\n            'deleted': [],\n            'modified': [],\n            'added': [],\n            'type_changed': []}\n\n    ar.add('first', git=True)\n    ar.add(opj('sub', 'third'))\n    ar.add('fifth')\n    sync_wrapper()\n\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # mixed annexed/not-annexed files ATm can't be committed with explicitly\n        # given paths in v6\n        # See:\n        # http://git-annex.branchable.com/bugs/committing_files_into_git_doesn__39__t_work_with_explicitly_given_paths_in_V6\n        ar.commit()\n    else:\n        super(AnnexRepo, ar).commit(files=['first', 'fifth', opj('sub', 'third'), 'second'])\n    eq_(stat, ar.get_status())\n\n    # remove a file in annex:\n    ar.remove('fifth')\n    stat['deleted'].append('fifth')\n    eq_(stat, ar.get_status())\n\n    # remove a file in git:\n    ar.remove('first')\n    stat['deleted'].append('first')\n    eq_(stat, ar.get_status())\n\n    # remove a submodule:\n    # rm; git rm; git commit\n    sub.precommit()  # Do precommit so there are not active batched processes etc\n    from datalad.utils import rmtree\n    rmtree(opj(ar.path, 'submod'))\n    stat['deleted'].append('submod')\n    eq_(stat, ar.get_status())\n    # recreate an empty mountpoint, since we currently do it in uninstall:\n    os.makedirs(opj(ar.path, 'submod'))\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n    ar.remove('submod')\n    # TODO: Why the difference?\n    stat['deleted'].append('submod')\n    stat['modified'].append('.gitmodules')\n    eq_(stat, ar.get_status())\n\n    # Note: Here again we need to use annex-proxy; This contradicts the addition\n    # of the very same submodule, which we needed to commit via\n    # -c core.bare=False instead. Otherwise the very same failure happens.\n    # Just vice versa. See above where 'submod' is added.\n    ar.commit(\"submod removed\", files=['submod', '.gitmodules'])\n    stat['modified'].remove('.gitmodules')\n    stat['deleted'].remove('submod')\n    eq_(stat, ar.get_status())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_status(path, path2):\n\n    ar = AnnexRepo(path, create=True)\n    _test_status(ar)\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        # in case of v6 have a second run with adjusted branch feature:\n        ar2 = AnnexRepo(path2, create=True)\n        ar2.commit(msg=\"empty commit to create branch 'master'\",\n                   options=['--allow-empty'])\n        ar2._run_annex_command('adjust', annex_options=['--unlock'])\n        _test_status(ar2)\n\n\n\n# TODO: test dirty\n# TODO: GitRep.dirty\n# TODO: test/utils ok_clean_git\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_set_remote_url(path):\n\n    ar = AnnexRepo(path, create=True)\n    ar.add_remote('some', 'http://example.com/.git')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://example.com/.git')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n    # change url:\n    ar.set_remote_url('some', 'http://believe.it')\n    assert_equal(ar.config['remote.some.url'],\n                 'http://believe.it')\n    assert_not_in('remote.some.annexurl', ar.config.keys())\n\n    # set push url:\n    ar.set_remote_url('some', 'ssh://whatever.ru', push=True)\n    assert_equal(ar.config['remote.some.pushurl'],\n                 'ssh://whatever.ru')\n    assert_in('remote.some.annexurl', ar.config.keys())\n    assert_equal(ar.config['remote.some.annexurl'],\n                 'ssh://whatever.ru')\n\n\n@with_tempfile(mkdir=True)\ndef test_wanted(path):\n    ar = AnnexRepo(path, create=True)\n    eq_(ar.get_preferred_content('wanted'), None)\n    # test samples with increasing \"trickiness\"\n    for v in (\"standard\",\n              \"include=*.nii.gz or include=*.nii\",\n              \"exclude=archive/* and (include=*.dat or smallerthan=2b)\"\n              ):\n        ar.set_preferred_content('wanted', expr=v)\n        eq_(ar.get_preferred_content('wanted'), v)\n    # give it some file so clone/checkout works without hiccups\n    create_tree(ar.path, {'1.dat': 'content'})\n    ar.add('1.dat')\n    ar.commit(msg=\"blah\")\n    # make a clone and see if all cool there\n    # intentionally clone as pure Git and do not annex init so to see if we\n    # are ignoring crummy log msgs\n    ar1_path = ar.path + '_1'\n    GitRepo.clone(ar.path, ar1_path)\n    ar1 = AnnexRepo(ar1_path, init=False)\n    eq_(ar1.get_preferred_content('wanted'), None)\n    eq_(ar1.get_preferred_content('wanted', 'origin'), v)\n    ar1.set_preferred_content('wanted', expr='standard')\n    eq_(ar1.get_preferred_content('wanted'), 'standard')\n\n\n@with_tempfile(mkdir=True)\ndef test_AnnexRepo_metadata(path):\n    # prelude\n    ar = AnnexRepo(path, create=True)\n    create_tree(\n        path,\n        {\n            'up.dat': 'content',\n            'd o\"w n': {\n                'd o w n.dat': 'lowcontent'\n            }\n        })\n    ar.add('.', git=False)\n    ar.commit('content')\n    ok_clean_git(path)\n    # fugue\n    # doesn't do anything if there is nothing to do\n    ar.set_metadata('up.dat')\n    eq_([], list(ar.get_metadata(None)))\n    eq_([], list(ar.get_metadata('')))\n    eq_([], list(ar.get_metadata([])))\n    eq_({'up.dat': {}}, dict(ar.get_metadata('up.dat')))\n    # basic invocation\n    eq_(1, len(list(ar.set_metadata(\n        'up.dat',\n        reset={'mike': 'awesome'},\n        add={'tag': 'awesome'},\n        remove={'tag': 'awesome'},  # cancels prev, just to use it\n        init={'virgin': 'true'},\n        purge=['nothere']))))\n    # no timestamps by default\n    md = dict(ar.get_metadata('up.dat'))\n    deq_({'up.dat': {\n        'virgin': ['true'],\n        'mike': ['awesome']}},\n        md)\n    # matching timestamp entries for all keys\n    md_ts = dict(ar.get_metadata('up.dat', timestamps=True))\n    for k in md['up.dat']:\n        assert_in('{}-lastchanged'.format(k), md_ts['up.dat'])\n    assert_in('lastchanged', md_ts['up.dat'])\n    # recursive needs a flag\n    assert_raises(CommandError, list, ar.set_metadata('.', purge=['virgin']))\n    list(ar.set_metadata('.', purge=['virgin'], recursive=True))\n    deq_({'up.dat': {\n        'mike': ['awesome']}},\n        dict(ar.get_metadata('up.dat')))\n    # Use trickier tags (spaces, =)\n    list(ar.set_metadata('.', reset={'tag': 'one and= '}, purge=['mike'], recursive=True))\n    playfile = opj('d o\"w n', 'd o w n.dat')\n    target = {\n        'up.dat': {\n            'tag': ['one and= ']},\n        playfile: {\n            'tag': ['one and= ']}}\n    deq_(target, dict(ar.get_metadata('.')))\n    # incremental work like a set\n    list(ar.set_metadata(playfile, add={'tag': 'one and= '}))\n    deq_(target, dict(ar.get_metadata('.')))\n    list(ar.set_metadata(playfile, add={'tag': ' two'}))\n    # returned values are sorted\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    # init honor prior values\n    list(ar.set_metadata(playfile, init={'tag': 'three'}))\n    eq_([' two', 'one and= '], dict(ar.get_metadata(playfile))[playfile]['tag'])\n    list(ar.set_metadata(playfile, remove={'tag': ' two'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # remove non-existing doesn't error and doesn't change anything\n    list(ar.set_metadata(playfile, remove={'ether': 'best'}))\n    deq_(target, dict(ar.get_metadata('.')))\n    # add works without prior existence\n    list(ar.set_metadata(playfile, add={'novel': 'best'}))\n    eq_(['best'], dict(ar.get_metadata(playfile))[playfile]['novel'])\n\n\n@with_tree(tree={'file.txt': 'content'})\n@serve_path_via_http()\n@with_tempfile\ndef test_AnnexRepo_addurl_batched_and_set_metadata(path, url, dest):\n    ar = AnnexRepo(dest, create=True)\n    fname = \"file.txt\"\n    ar.add_url_to_file(fname, urljoin(url, fname), batch=True)\n    list(ar.set_metadata(fname, init={\"number\": \"one\"}))\n    eq_([\"one\"], dict(ar.get_metadata(fname))[fname][\"number\"])\n\n\n@with_tempfile(mkdir=True)\ndef test_change_description(path):\n    # prelude\n    ar = AnnexRepo(path, create=True, description='some')\n    eq_(ar.get_description(), 'some')\n    # try change it\n    ar = AnnexRepo(path, create=False, init=True, description='someother')\n    # this doesn't cut the mustard, still old\n    eq_(ar.get_description(), 'some')\n    # need to resort to \"internal\" helper\n    ar._init(description='someother')\n    eq_(ar.get_description(), 'someother')\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_corresponding_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we should be on master or a corresponding branch like annex/direct/master\n    # respectively if ran in direct mode build.\n    # We want to get 'master' in any case\n    eq_('master', ar.get_corresponding_branch())\n\n    # special case v6 adjusted branch is not provided by a dedicated build:\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        # as above, we still want to get 'master', while being on\n        # 'adjusted/master(unlocked)'\n        eq_('adjusted/master(unlocked)', ar.get_active_branch())\n        eq_('master', ar.get_corresponding_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_get_tracking_branch(path):\n\n    ar = AnnexRepo(path)\n\n    # we want the relation to original branch, especially in direct mode\n    # or even in v6 adjusted branch\n    eq_(('origin', 'refs/heads/master'), ar.get_tracking_branch())\n\n\n@with_testrepos('basic_annex', flavors=['clone'])\ndef test_AnnexRepo_is_managed_branch(path):\n\n    ar = AnnexRepo(path)\n\n    if ar.is_direct_mode():\n        ok_(ar.is_managed_branch())\n    else:\n        # ATM only direct mode and v6 adjusted branches should return True.\n        # Adjusted branch requires a call of git-annex-adjust and shouldn't\n        # be the state of a fresh clone\n        ok_(not ar.is_managed_branch())\n    if ar.config.getint(\"annex\", \"version\") == 6:\n        ar.adjust()\n        ok_(ar.is_managed_branch())\n\n\n@with_tempfile(mkdir=True)\n@with_tempfile()\ndef test_AnnexRepo_flyweight_monitoring_inode(path, store):\n    # testing for issue #1512\n    check_repo_deals_with_inode_change(AnnexRepo, path, store)\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_is_not_special(path):\n    ar = AnnexRepo(path, create=True)\n    # doesn't exist -- we fail by default\n    assert_raises(RemoteNotAvailableError, ar.is_special_annex_remote, \"fake\")\n    assert_false(ar.is_special_annex_remote(\"fake\", check_if_known=False))\n\n\n@with_tempfile(mkdir=True)\ndef test_fake_dates(path):\n    ar = AnnexRepo(path, create=True, fake_dates=True)\n    timestamp = ar.config.obtain(\"datalad.fake-dates-start\") + 1\n    # Commits from the \"git annex init\" call are one second ahead.\n    for commit in ar.get_branch_commits(\"git-annex\"):\n        eq_(timestamp, commit.committed_date)\n    assert_in(\"timestamp={}s\".format(timestamp),\n              ar.repo.git.cat_file(\"blob\", \"git-annex:uuid.log\"))\n\n\ndef test_get_size_from_perc_complete():\n    f = _get_size_from_perc_complete\n    eq_(f(0, 0), 0)\n    eq_(f(0, '0'), 0)\n    eq_(f(100, '0'), 0)  # we do not know better\n    eq_(f(1, '1'), 100)\n    # with no percentage info, we don't know better either:\n    eq_(f(1, ''), 0)\n\n\n# to prevent regression\n# http://git-annex.branchable.com/bugs/v6_-_under_subdir__58___git_add___34__whines__34____44___git_commit___34__blows__34__/\n# It is disabled because is not per se relevant to DataLad since we do not\n# Since we invoke from the top of the repo, we do not hit it,\n# but thought to leave it around if we want to enforce/test system-wide git being\n# compatible with annex for v6 mode\n@with_tempfile(mkdir=True)\ndef _test_add_under_subdir(path):\n    ar = AnnexRepo(path, create=True, version=6)\n    gr = GitRepo(path)  # \"Git\" view over the repository, so we force \"git add\"\n    subdir = opj(path, 'sub')\n    subfile = opj('sub', 'empty')\n    # os.mkdir(subdir)\n    create_tree(subdir, {'empty': ''})\n    runner = Runner(cwd=subdir)\n    with chpwd(subdir):\n        runner(['git', 'add', 'empty'])  # should add sucesfully\n        # gr.commit('important') #\n        runner(['git', 'commit', '-m', 'important'])\n        ar.is_under_annex(subfile)\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.10.3.1.dev91\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 124ms\nInstalled 25 packages in 5.07s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Za93z/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Za93z/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Za93z/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Za93z/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmp1Za93z/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpCcF3UE/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpCcF3UE/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpCcF3UE/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpCcF3UE/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpCcF3UE/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 51, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/datalad/__init__.py\", line 43, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/datalad/cmd.py\", line 28, in <module>\n    from .support import path as op\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/datalad/support/path.py\", line 19, in <module>\n    from ..utils import (\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/datalad/utils.py\", line 25, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 230ms\nInstalled 26 packages in 2.34s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 57 packages in 1.40s\nPrepared 1 package in 1.25s\nInstalled 50 packages in 48ms\n + appdirs==1.4.4\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + cryptography==44.0.0\n + datalad==0.10.3.1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9)\n + deprecated==1.2.15\n + exifread==3.0.0\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.6.0\n + idna==3.10\n + importlib-resources==5.12.0\n + iso8601==2.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + patool==1.12\n + pillow==9.5.0\n + pycparser==2.21\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + requests==2.31.0\n + requests-ftp==0.3.1\n + secretstorage==3.3.3\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.4.1\n + tqdm==4.67.1\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\nResolved 90 packages in 166ms\nInstalled 31 packages in 74ms\n + alabaster==0.7.13\n + asv==0.6.4\n + asv-runner==0.2.1\n + babel==2.14.0\n + build==1.1.1\n + distlib==0.3.9\n + docutils==0.19\n + filelock==3.12.2\n + imagesize==1.4.1\n + jinja2==3.1.5\n + json5==0.9.16\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + platformdirs==4.0.0\n + psutil==6.1.1\n + pygments==2.17.2\n + pympler==1.1\n + pypandoc==1.14\n + pyproject-hooks==1.2.0\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n + tabulate==0.9.0\n + virtualenv==20.26.6\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9\nplugins: cov-4.1.0\ncollected 61 items\n\nr2e_tests/test_1.py EFFEFFEEEEFxxEFFExEEFEFFFFExEEFEFEEEFxFF.EEEEssFEFFF [ 85%]\nEFFFFEFF.                                                                [100%]\n\n==================================== ERRORS ====================================\n_____________ ERROR at setup of test_AnnexRepo_instance_from_clone _____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 107\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_instance_from_clone(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:107\n_____________ ERROR at setup of test_AnnexRepo_crippled_filesystem _____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 148\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_crippled_filesystem(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:148\n_______________ ERROR at setup of test_AnnexRepo_set_direct_mode _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 197\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_set_direct_mode(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:197\n_________________ ERROR at setup of test_AnnexRepo_annex_proxy _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 221\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_annex_proxy(src, annex_path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:221\n________________ ERROR at setup of test_AnnexRepo_get_file_key _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 241\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_get_file_key(src, annex_path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:241\n_______________ ERROR at setup of test_AnnexRepo_get_outofspace ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 270\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_get_outofspace(annex_path):\nE       fixture 'annex_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:270\n_________________ ERROR at setup of test_AnnexRepo_web_remote __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 368\n  @with_tree(tree=(('about.txt', 'Lots of abouts'),\n                   ('about2.txt', 'more abouts'),\n                   ('d', {'sub.txt': 'more stuff'})))\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_web_remote(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:368\n_____________ ERROR at setup of test_AnnexRepo_migrating_backends ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 568\n  @with_testrepos('.*annex.*', flavors=['local', 'network'])\n  @with_tempfile\n  def test_AnnexRepo_migrating_backends(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:568\n_______________ ERROR at setup of test_AnnexRepo_backend_option ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 651\n  @with_tree(**tree1args)\n  @serve_path_via_http()\n  def test_AnnexRepo_backend_option(path, url):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:651\n______________ ERROR at setup of test_AnnexRepo_get_file_backend _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 677\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_get_file_backend(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:677\n______________ ERROR at setup of test_AnnexRepo_on_uninited_annex ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 761\n  @with_testrepos('basic_annex', flavors=['local'])\n  @with_tempfile\n  def test_AnnexRepo_on_uninited_annex(origin, path):\nE       fixture 'origin' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:761\n_____________________ ERROR at setup of test_AnnexRepo_get _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 921\n  @ignore_nose_capturing_stdout\n  @with_testrepos('.*annex.*', flavors=['local'])\n  # TODO: flavor 'network' has wrong content for test-annex.dat!\n  @with_tempfile\n  def test_AnnexRepo_get(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:921\n___________ ERROR at setup of test_AnnexRepo_addurl_to_file_batched ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1004\n  @with_tree(tree=(('about.txt', 'Lots of abouts'),\n                   ('about2.txt', 'more abouts'),\n                   ('about2_.txt', 'more abouts_'),\n                   ('d', {'sub.txt': 'more stuff'})))\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_addurl_to_file_batched(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1004\n_________ ERROR at setup of test_annexrepo_fake_dates_disables_batched _________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1113\n  @with_tree(tree={\"foo\": \"foo content\"})\n  @serve_path_via_http()\n  @with_tree(tree={\"bar\": \"bar content\"})\n  def test_annexrepo_fake_dates_disables_batched(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1113\n_______________________ ERROR at setup of test_annex_ssh _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1152\n  @skip_ssh\n  @with_tempfile\n  @with_testrepos('basic_annex', flavors=['local'])\n  @with_testrepos('basic_annex', flavors=['local'])\n  def test_annex_ssh(repo_path, remote_1_path, remote_2_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1152\n_____________________ ERROR at setup of test_repo_version ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1275\n  @with_tempfile\n  @with_tempfile\n  @with_tempfile\n  def test_repo_version(path1, path2, path3):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1275\n_____________________ ERROR at setup of test_annex_copy_to _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1296\n  @with_testrepos('.*annex.*', flavors=['clone'])\n  @with_tempfile(mkdir=True)\n  def test_annex_copy_to(origin, clone):\nE       fixture 'origin' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1296\n______________________ ERROR at setup of test_annex_drop _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1369\n  @with_testrepos('.*annex.*', flavors=['local'])\n  # TODO: flavor 'network' has wrong content for test-annex.dat!\n  @with_tempfile\n  def test_annex_drop(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1369\n____________________ ERROR at setup of test_get_description ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1665\n  @with_tempfile\n  @with_tempfile\n  def test_get_description(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1665\n__________________ ERROR at setup of test_AnnexRepo_flyweight __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1696\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_flyweight(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1696\n_________________ ERROR at setup of test_AnnexRepo_get_toppath _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1725\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile(mkdir=True)\n  @with_tempfile\n  def test_AnnexRepo_get_toppath(repo, tempdir, repo2):\nE       fixture 'repo' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1725\n________________ ERROR at setup of test_AnnexRepo_add_submodule ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1744\n  @with_testrepos(\".*basic.*\", flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_add_submodule(source, path):\nE       fixture 'source' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1744\n___________________ ERROR at setup of test_AnnexRepo_status ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2112\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_status(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2112\n_______ ERROR at setup of test_AnnexRepo_addurl_batched_and_set_metadata _______\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2258\n  @with_tree(tree={'file.txt': 'content'})\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_addurl_batched_and_set_metadata(path, url, dest):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2258\n_________ ERROR at setup of test_AnnexRepo_flyweight_monitoring_inode __________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2329\n  @with_tempfile(mkdir=True)\n  @with_tempfile()\n  def test_AnnexRepo_flyweight_monitoring_inode(path, store):\nE       fixture 'store' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2329\n=================================== FAILURES ===================================\n____________________ test_AnnexRepo_instance_from_existing _____________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexg740gdmf'}\nio = <module 'io' from '/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/io.py'>\n\n    @make_decorator(func)\n    def newfunc(*args, **kwargs):\n        import io\n        try:\n>           func(*args, **kwargs)\n\ndatalad/tests/utils.py:1279: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1090: in newfunc\n    reraise(*exc_info)\n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\ndatalad/tests/utils.py:749: in newfunc\n    testrepos_uris = _get_testrepos_uris(regex, flavors_)\ndatalad/tests/utils.py:685: in _get_testrepos_uris\n    _basic_annex_test_repo.create()\ndatalad/tests/utils_testrepos.py:92: in create\n    self.populate()\ndatalad/tests/utils_testrepos.py:106: in populate\n    self.create_info_file()\ndatalad/tests/utils_testrepos.py:132: in create_info_file\n    annex=False)\ndatalad/tests/utils_testrepos.py:85: in create_file\n    self.repo.add(name, git=True)\ndatalad/support/gitrepo.py:312: in newfunc\n    result = func(self, files_new, *args, **kwargs)\ndatalad/support/annexrepo.py:1456: in add\n    git_options=git_options, annex_options=annex_options, update=update\ndatalad/support/annexrepo.py:1572: in add_\n    expect_stderr=True):\ndatalad/support/annexrepo.py:2463: in _run_annex_command_json\n    raise e\ndatalad/support/annexrepo.py:2379: in _run_annex_command_json\n    **kwargs)\ndatalad/support/annexrepo.py:1089: in _run_annex_command\n    raise e\ndatalad/support/annexrepo.py:1081: in _run_annex_command\n    return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\ndatalad/cmd.py:673: in run\n    cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.cmd.GitRunner object at 0x723cff5dd910>\ncmd = ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', ...]\nlog_stdout = <datalad.support.annexrepo.ProcessAnnexProgressIndicators object at 0x723cff66fc50>\nlog_stderr = 'offline', log_online = True, expect_stderr = True\nexpect_fail = True, cwd = None\nenv = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nshell = False, stdin = None\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n    \n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n    \n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n    \n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n    \n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n    \n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n    \n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n    \n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n    \n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n    \n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n    \n        env : string, optional\n            Custom environment to pass\n    \n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n    \n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n    \n        Returns\n        -------\n        (stdout, stderr) - bytes!\n    \n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n    \n        popen_env = env or self.env\n    \n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n    \n        if self.protocol.do_execute_ext_commands:\n    \n            if shell is None:\n                shell = isinstance(cmd, string_types)\n    \n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n    \n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n    \n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n    \n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n    \n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n    \n                status = proc.poll()\n    \n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n    \n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d.%s%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status,\n                           \"\" if log_online else \" out=%s\" % out[0],\n                           \"\" if log_online else \" err=%s\" % out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n>                   raise CommandError(str(cmd), msg, status, out[0], out[1])\nE                   datalad.support.exceptions.CommandError: CommandError: command '['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt']' failed with exitcode 1\nE                   Failed to run ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_l8etlpik'. Exit code=1.\n\ndatalad/cmd.py:533: CommandError\n______________________ test_AnnexRepo_instance_brand_new _______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexm5s9vy8a'}\nio = <module 'io' from '/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/io.py'>\n\n    @make_decorator(func)\n    def newfunc(*args, **kwargs):\n        import io\n        try:\n>           func(*args, **kwargs)\n\ndatalad/tests/utils.py:1279: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1090: in newfunc\n    reraise(*exc_info)\n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexm5s9vy8a'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_instance_brand_newcp9ogpgq'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_instance_brand_new() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n________________________ test_AnnexRepo_is_direct_mode _________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annex8bg7w0ca'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\nexc_info = (<class 'TypeError'>, TypeError(\"test_AnnexRepo_is_direct_mode() got multiple values for argument 'path'\"), <traceback object at 0x723cff6eb550>)\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        try:\n            func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           reraise(*exc_info)\n\ndatalad/tests/utils.py:1090: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annex8bg7w0ca'}, flavors_ = ['local']\ntestrepos_uris = ['/tmp/datalad_temp_testrepo_l8etlpik', '/tmp/datalad_temp_testrepo_fsgonntv', '/tmp/datalad_temp_testrepo_asgd26el']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_testrepo_l8etlpik'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_is_direct_mode() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n____________________ test_AnnexRepo_is_direct_mode_gitrepo _____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexxcquvch7'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_is_direct_mode_gitrepo3vt0pmmo'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_is_direct_mode_gitrepo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_get_remote_na _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexwlwx6p9p'}, flavors_ = ['local']\ntestrepos_uris = ['/tmp/datalad_temp_testrepo_l8etlpik'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_testrepo_l8etlpik'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_remote_na() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_________________________ test_find_batch_equivalence __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexy1703vqb'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_find_batch_equivalence'}\nd = '/tmp/datalad_temp_tree_test_find_batch_equivalenced2q6r9jj'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_find_batch_equivalence() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:418: TypeError\n________________________________ test_repo_info ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexehdey2p5'}\nfilename = '/tmp/datalad_temp_test_repo_info2isfo7je'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_repo_info() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_always_commit _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexgwfyo1zx'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_always_commitbwwglmy7'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_always_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n____________________________ test_AnnexRepo_commit _____________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexyveai97g'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\nexc_info = (<class 'TypeError'>, TypeError(\"test_AnnexRepo_commit() got multiple values for argument 'path'\"), <traceback object at 0x723cff02fc30>)\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        try:\n            func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           reraise(*exc_info)\n\ndatalad/tests/utils.py:1090: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexyveai97g'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_committfz84ngk'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_add_to_annex __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex43rn8hni'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_i9tt3z6e', '/tmp/datalad_temp_clone_url_dkiizirq', '/tmp/datalad_temp_clone_url_ods19ybx']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_clone_url_i9tt3z6e'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_to_annex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n__________________________ test_AnnexRepo_add_to_git ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexk493dcdg'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_rin8egww', '/tmp/datalad_temp_clone_url_i_lblow2', '/tmp/datalad_temp_clone_url_37dbx83c']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_clone_url_rin8egww'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_to_git() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n__________________ test_AnnexRepo_add_unexpected_direct_mode ___________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexi9lh_u94'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_ryqlx2l6'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_ryqlx2l6'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_unexpected_direct_mode() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_____________________________ test_annex_backends ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex1r4v_px_'}\nfilename = '/tmp/datalad_temp_test_annex_backendssk5rouu_'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_backends() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n______________________________ test_annex_remove _______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexji0iuhkn'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_c60n5hq_'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_c60n5hq_'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_annex_remove() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_________________________ test_annex_get_annexed_files _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexjnv0ujca'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_annex_get_annexed_files'}\nd = '/tmp/datalad_temp_tree_test_annex_get_annexed_files0rkkxtte'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_annex_get_annexed_files() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:418: TypeError\n__________________________ test_annex_add_no_dotfiles __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexq0057f21'}\nfilename = '/tmp/datalad_temp_test_annex_add_no_dotfiles5q47o4n1'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_add_no_dotfiles() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_annex_version_handling __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2vkwy8e1'}\nfilename = '/tmp/datalad_temp_test_annex_version_handlingyc9xpojo'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_version_handling() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_____________________________ test_AnnexRepo_dirty _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex0np9emwn'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_dirty8j4u5zu0'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n________________________ test_AnnexRepo_set_remote_url _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexnc_ww5zo'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_set_remote_urlnsajfox_'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_set_remote_url() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________________ test_wanted __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexonx8zub5'}\nfilename = '/tmp/datalad_temp_test_wanted112h29ps'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_wanted() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________________ test_AnnexRepo_metadata ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexlsf38s4a'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_metadata2msvqmvv'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_metadata() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________________ test_change_description ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex9ngzzv_u'}\nfilename = '/tmp/datalad_temp_test_change_descriptionnanis753'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_change_description() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________ test_AnnexRepo_get_corresponding_branch ____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexh6y6abuh'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_mcd9b4yy'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_mcd9b4yy'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_corresponding_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n______________________ test_AnnexRepo_get_tracking_branch ______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex0dkxkinj'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_0wkts9f1'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_0wkts9f1'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_tracking_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_______________________ test_AnnexRepo_is_managed_branch _______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvpzn48dc'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_wp2a6ceg'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_wp2a6ceg'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_is_managed_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n___________________________ test_fake_is_not_special ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex6v9xx66t'}\nfilename = '/tmp/datalad_temp_test_fake_is_not_specialixcwb10m'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_fake_is_not_special() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_______________________________ test_fake_dates ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexlkj_o0p1'}\nfilename = '/tmp/datalad_temp_test_fake_datesbpbxauua'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_fake_dates() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n==================================== PASSES ====================================\n_____________________ test_ProcessAnnexProgressIndicators ______________________\n----------------------------- Captured stdout call -----------------------------\n\nfile1:   0%|          | 0.00/100 [00:00<?, ?B/s]\n                                                \n\nTotal:   0%|          | 0.00/100 [00:00<?, ?B/s]\nTotal (1 ok out of 2): 100%|██████████| 100/100 [00:00<00:00, 4.60kB/s]\n                                                                       \n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_ProcessAnnexProgressIndicators\nPASSED r2e_tests/test_1.py::test_get_size_from_perc_complete\nSKIPPED [2] .venv/lib/python3.7/site-packages/_pytest/unittest.py:371: TODO\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_file_has_content - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_file_has_content will be ignored\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_is_under_annex - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_is_under_annex will be ignored\nXFAIL r2e_tests/test_1.py::test_dropkey - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_dropkey will be ignored\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_get_contentlocation - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_get_contentlocation will be ignored\nXFAIL r2e_tests/test_1.py::test_is_available - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_is_available will be ignored\nERROR r2e_tests/test_1.py::test_AnnexRepo_instance_from_clone\nERROR r2e_tests/test_1.py::test_AnnexRepo_crippled_filesystem\nERROR r2e_tests/test_1.py::test_AnnexRepo_set_direct_mode\nERROR r2e_tests/test_1.py::test_AnnexRepo_annex_proxy\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_file_key\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_outofspace\nERROR r2e_tests/test_1.py::test_AnnexRepo_web_remote\nERROR r2e_tests/test_1.py::test_AnnexRepo_migrating_backends\nERROR r2e_tests/test_1.py::test_AnnexRepo_backend_option\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_file_backend\nERROR r2e_tests/test_1.py::test_AnnexRepo_on_uninited_annex\nERROR r2e_tests/test_1.py::test_AnnexRepo_get\nERROR r2e_tests/test_1.py::test_AnnexRepo_addurl_to_file_batched\nERROR r2e_tests/test_1.py::test_annexrepo_fake_dates_disables_batched\nERROR r2e_tests/test_1.py::test_annex_ssh\nERROR r2e_tests/test_1.py::test_repo_version\nERROR r2e_tests/test_1.py::test_annex_copy_to\nERROR r2e_tests/test_1.py::test_annex_drop\nERROR r2e_tests/test_1.py::test_get_description\nERROR r2e_tests/test_1.py::test_AnnexRepo_flyweight\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_toppath\nERROR r2e_tests/test_1.py::test_AnnexRepo_add_submodule\nERROR r2e_tests/test_1.py::test_AnnexRepo_status\nERROR r2e_tests/test_1.py::test_AnnexRepo_addurl_batched_and_set_metadata\nERROR r2e_tests/test_1.py::test_AnnexRepo_flyweight_monitoring_inode\nFAILED r2e_tests/test_1.py::test_AnnexRepo_instance_from_existing - datalad.s...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_instance_brand_new - TypeError: te...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_direct_mode - TypeError: test_A...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_direct_mode_gitrepo - TypeError...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_remote_na - TypeError: test_An...\nFAILED r2e_tests/test_1.py::test_find_batch_equivalence - TypeError: test_fin...\nFAILED r2e_tests/test_1.py::test_repo_info - TypeError: test_repo_info() got ...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_always_commit - TypeError: test_An...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_commit - TypeError: test_AnnexRepo...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_to_annex - TypeError: test_Ann...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_to_git - TypeError: test_Annex...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_unexpected_direct_mode - TypeE...\nFAILED r2e_tests/test_1.py::test_annex_backends - TypeError: test_annex_backe...\nFAILED r2e_tests/test_1.py::test_annex_remove - TypeError: test_annex_remove(...\nFAILED r2e_tests/test_1.py::test_annex_get_annexed_files - TypeError: test_an...\nFAILED r2e_tests/test_1.py::test_annex_add_no_dotfiles - TypeError: test_anne...\nFAILED r2e_tests/test_1.py::test_annex_version_handling - TypeError: test_ann...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_dirty - TypeError: test_AnnexRepo_...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_set_remote_url - TypeError: test_A...\nFAILED r2e_tests/test_1.py::test_wanted - TypeError: test_wanted() got multip...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_metadata - TypeError: test_AnnexRe...\nFAILED r2e_tests/test_1.py::test_change_description - TypeError: test_change_...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_corresponding_branch - TypeErr...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_tracking_branch - TypeError: t...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_managed_branch - TypeError: tes...\nFAILED r2e_tests/test_1.py::test_fake_is_not_special - TypeError: test_fake_i...\nFAILED r2e_tests/test_1.py::test_fake_dates - TypeError: test_fake_dates() go...\n======== 27 failed, 2 passed, 2 skipped, 5 xfailed, 25 errors in 4.46s =========\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9\nplugins: cov-4.1.0\ncollected 61 items\n\nr2e_tests/test_1.py EFFEFFEEEEFxxEFFExEEFEFFFFExEEFEFEEEFxFF.EEEEssFEFFF [ 85%]\nEFFFFEFFF                                                                [100%]\n\n==================================== ERRORS ====================================\n_____________ ERROR at setup of test_AnnexRepo_instance_from_clone _____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 107\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_instance_from_clone(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:107\n_____________ ERROR at setup of test_AnnexRepo_crippled_filesystem _____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 148\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_crippled_filesystem(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:148\n_______________ ERROR at setup of test_AnnexRepo_set_direct_mode _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 197\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*')\n  @with_tempfile\n  def test_AnnexRepo_set_direct_mode(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:197\n_________________ ERROR at setup of test_AnnexRepo_annex_proxy _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 221\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_annex_proxy(src, annex_path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:221\n________________ ERROR at setup of test_AnnexRepo_get_file_key _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 241\n  @assert_cwd_unchanged\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_get_file_key(src, annex_path):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:241\n_______________ ERROR at setup of test_AnnexRepo_get_outofspace ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 270\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_get_outofspace(annex_path):\nE       fixture 'annex_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:270\n_________________ ERROR at setup of test_AnnexRepo_web_remote __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 368\n  @with_tree(tree=(('about.txt', 'Lots of abouts'),\n                   ('about2.txt', 'more abouts'),\n                   ('d', {'sub.txt': 'more stuff'})))\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_web_remote(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:368\n_____________ ERROR at setup of test_AnnexRepo_migrating_backends ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 568\n  @with_testrepos('.*annex.*', flavors=['local', 'network'])\n  @with_tempfile\n  def test_AnnexRepo_migrating_backends(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:568\n_______________ ERROR at setup of test_AnnexRepo_backend_option ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 651\n  @with_tree(**tree1args)\n  @serve_path_via_http()\n  def test_AnnexRepo_backend_option(path, url):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:651\n______________ ERROR at setup of test_AnnexRepo_get_file_backend _______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 677\n  @with_testrepos('.*annex.*', flavors=local_testrepo_flavors)\n  @with_tempfile\n  def test_AnnexRepo_get_file_backend(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:677\n______________ ERROR at setup of test_AnnexRepo_on_uninited_annex ______________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 761\n  @with_testrepos('basic_annex', flavors=['local'])\n  @with_tempfile\n  def test_AnnexRepo_on_uninited_annex(origin, path):\nE       fixture 'origin' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:761\n_____________________ ERROR at setup of test_AnnexRepo_get _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 921\n  @ignore_nose_capturing_stdout\n  @with_testrepos('.*annex.*', flavors=['local'])\n  # TODO: flavor 'network' has wrong content for test-annex.dat!\n  @with_tempfile\n  def test_AnnexRepo_get(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:921\n___________ ERROR at setup of test_AnnexRepo_addurl_to_file_batched ____________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1004\n  @with_tree(tree=(('about.txt', 'Lots of abouts'),\n                   ('about2.txt', 'more abouts'),\n                   ('about2_.txt', 'more abouts_'),\n                   ('d', {'sub.txt': 'more stuff'})))\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_addurl_to_file_batched(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1004\n_________ ERROR at setup of test_annexrepo_fake_dates_disables_batched _________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1113\n  @with_tree(tree={\"foo\": \"foo content\"})\n  @serve_path_via_http()\n  @with_tree(tree={\"bar\": \"bar content\"})\n  def test_annexrepo_fake_dates_disables_batched(sitepath, siteurl, dst):\nE       fixture 'sitepath' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1113\n_______________________ ERROR at setup of test_annex_ssh _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1152\n  @skip_ssh\n  @with_tempfile\n  @with_testrepos('basic_annex', flavors=['local'])\n  @with_testrepos('basic_annex', flavors=['local'])\n  def test_annex_ssh(repo_path, remote_1_path, remote_2_path):\nE       fixture 'repo_path' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1152\n_____________________ ERROR at setup of test_repo_version ______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1275\n  @with_tempfile\n  @with_tempfile\n  @with_tempfile\n  def test_repo_version(path1, path2, path3):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1275\n_____________________ ERROR at setup of test_annex_copy_to _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1296\n  @with_testrepos('.*annex.*', flavors=['clone'])\n  @with_tempfile(mkdir=True)\n  def test_annex_copy_to(origin, clone):\nE       fixture 'origin' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1296\n______________________ ERROR at setup of test_annex_drop _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1369\n  @with_testrepos('.*annex.*', flavors=['local'])\n  # TODO: flavor 'network' has wrong content for test-annex.dat!\n  @with_tempfile\n  def test_annex_drop(src, dst):\nE       fixture 'src' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1369\n____________________ ERROR at setup of test_get_description ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1665\n  @with_tempfile\n  @with_tempfile\n  def test_get_description(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1665\n__________________ ERROR at setup of test_AnnexRepo_flyweight __________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1696\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_flyweight(path1, path2):\nE       fixture 'path1' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1696\n_________________ ERROR at setup of test_AnnexRepo_get_toppath _________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1725\n  @with_testrepos(flavors=local_testrepo_flavors)\n  @with_tempfile(mkdir=True)\n  @with_tempfile\n  def test_AnnexRepo_get_toppath(repo, tempdir, repo2):\nE       fixture 'repo' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1725\n________________ ERROR at setup of test_AnnexRepo_add_submodule ________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 1744\n  @with_testrepos(\".*basic.*\", flavors=['local'])\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_add_submodule(source, path):\nE       fixture 'source' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:1744\n___________________ ERROR at setup of test_AnnexRepo_status ____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2112\n  @with_tempfile(mkdir=True)\n  @with_tempfile(mkdir=True)\n  def test_AnnexRepo_status(path, path2):\nE       fixture 'path2' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2112\n_______ ERROR at setup of test_AnnexRepo_addurl_batched_and_set_metadata _______\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2258\n  @with_tree(tree={'file.txt': 'content'})\n  @serve_path_via_http()\n  @with_tempfile\n  def test_AnnexRepo_addurl_batched_and_set_metadata(path, url, dest):\nE       fixture 'url' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2258\n_________ ERROR at setup of test_AnnexRepo_flyweight_monitoring_inode __________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py, line 2329\n  @with_tempfile(mkdir=True)\n  @with_tempfile()\n  def test_AnnexRepo_flyweight_monitoring_inode(path, store):\nE       fixture 'store' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9/r2e_tests/test_1.py:2329\n=================================== FAILURES ===================================\n____________________ test_AnnexRepo_instance_from_existing _____________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annex6zjrw_pd'}\nio = <module 'io' from '/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/io.py'>\n\n    @make_decorator(func)\n    def newfunc(*args, **kwargs):\n        import io\n        try:\n>           func(*args, **kwargs)\n\ndatalad/tests/utils.py:1279: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1090: in newfunc\n    reraise(*exc_info)\n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\ndatalad/tests/utils.py:749: in newfunc\n    testrepos_uris = _get_testrepos_uris(regex, flavors_)\ndatalad/tests/utils.py:685: in _get_testrepos_uris\n    _basic_annex_test_repo.create()\ndatalad/tests/utils_testrepos.py:92: in create\n    self.populate()\ndatalad/tests/utils_testrepos.py:106: in populate\n    self.create_info_file()\ndatalad/tests/utils_testrepos.py:132: in create_info_file\n    annex=False)\ndatalad/tests/utils_testrepos.py:85: in create_file\n    self.repo.add(name, git=True)\ndatalad/support/gitrepo.py:312: in newfunc\n    result = func(self, files_new, *args, **kwargs)\ndatalad/support/annexrepo.py:1456: in add\n    git_options=git_options, annex_options=annex_options, update=update\ndatalad/support/annexrepo.py:1572: in add_\n    expect_stderr=True):\ndatalad/support/annexrepo.py:2463: in _run_annex_command_json\n    raise e\ndatalad/support/annexrepo.py:2379: in _run_annex_command_json\n    **kwargs)\ndatalad/support/annexrepo.py:1089: in _run_annex_command\n    raise e\ndatalad/support/annexrepo.py:1081: in _run_annex_command\n    return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\ndatalad/cmd.py:673: in run\n    cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <datalad.cmd.GitRunner object at 0x75ceb468df30>\ncmd = ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', ...]\nlog_stdout = <datalad.support.annexrepo.ProcessAnnexProgressIndicators object at 0x75ceb496ead0>\nlog_stderr = 'offline', log_online = True, expect_stderr = True\nexpect_fail = True, cwd = None\nenv = {'ANTHROPIC_API_KEY': 'sk-ant-api03-kztdHBP1RruInQ2uzw8m4wJLbhPHBrb-qHX2mhAKmwoMFvM_dI2oWb5V6JtS45MScuW5CCjPusIrt9Zatx...3b4166bef66ef252935c/server/bin/helpers/browser.sh', 'COHERE_API_KEY': '66TDU3MmN5ELJT1NHWklGG2PJjK2zwVSQbVq42ol', ...}\nshell = False, stdin = None\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n    \n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n    \n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n    \n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n    \n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n    \n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n    \n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n    \n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n    \n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n    \n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n    \n        env : string, optional\n            Custom environment to pass\n    \n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n    \n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n    \n        Returns\n        -------\n        (stdout, stderr) - bytes!\n    \n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n    \n        popen_env = env or self.env\n    \n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n    \n        if self.protocol.do_execute_ext_commands:\n    \n            if shell is None:\n                shell = isinstance(cmd, string_types)\n    \n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n    \n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n    \n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n    \n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n    \n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n    \n                status = proc.poll()\n    \n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n    \n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d.%s%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status,\n                           \"\" if log_online else \" out=%s\" % out[0],\n                           \"\" if log_online else \" err=%s\" % out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n>                   raise CommandError(str(cmd), msg, status, out[0], out[1])\nE                   datalad.support.exceptions.CommandError: CommandError: command '['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt']' failed with exitcode 1\nE                   Failed to run ['git', '-c', 'receive.autogc=0', '-c', 'gc.auto=0', 'annex', 'add', '--json', '-c', 'annex.largefiles=nothing', '--include-dotfiles', '--', 'INFO.txt'] under '/tmp/datalad_temp_testrepo_p502jaiu'. Exit code=1.\n\ndatalad/cmd.py:533: CommandError\n______________________ test_AnnexRepo_instance_brand_new _______________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexue4xnpwx'}\nio = <module 'io' from '/home/gcpuser/.local/share/uv/python/cpython-3.7.9-linux-x86_64-gnu/lib/python3.7/io.py'>\n\n    @make_decorator(func)\n    def newfunc(*args, **kwargs):\n        import io\n        try:\n>           func(*args, **kwargs)\n\ndatalad/tests/utils.py:1279: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \ndatalad/tests/utils.py:1090: in newfunc\n    reraise(*exc_info)\n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexue4xnpwx'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_instance_brand_newpth3lrmx'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_instance_brand_new() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n________________________ test_AnnexRepo_is_direct_mode _________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annex6vez5rrr'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\nexc_info = (<class 'TypeError'>, TypeError(\"test_AnnexRepo_is_direct_mode() got multiple values for argument 'path'\"), <traceback object at 0x75ceb4862140>)\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        try:\n            func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           reraise(*exc_info)\n\ndatalad/tests/utils.py:1090: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annex6vez5rrr'}, flavors_ = ['local']\ntestrepos_uris = ['/tmp/datalad_temp_testrepo_p502jaiu', '/tmp/datalad_temp_testrepo_1x5l6oq7', '/tmp/datalad_temp_testrepo_8f5r91fv']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_testrepo_p502jaiu'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_is_direct_mode() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n____________________ test_AnnexRepo_is_direct_mode_gitrepo _____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexdbvnsgoy'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_is_direct_mode_gitrepo74yoayyw'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_is_direct_mode_gitrepo() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_get_remote_na _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexufyqm3im'}, flavors_ = ['local']\ntestrepos_uris = ['/tmp/datalad_temp_testrepo_p502jaiu'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_testrepo_p502jaiu'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_remote_na() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_________________________ test_find_batch_equivalence __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex8rsm0viv'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_find_batch_equivalence'}\nd = '/tmp/datalad_temp_tree_test_find_batch_equivalenceqrsbstop'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_find_batch_equivalence() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:418: TypeError\n________________________________ test_repo_info ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex2nh3q9zp'}\nfilename = '/tmp/datalad_temp_test_repo_infozah_8ck8'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_repo_info() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_always_commit _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex791cyr98'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_always_commitxtrb0xa7'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_always_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n____________________________ test_AnnexRepo_commit _____________________________\n\nargs = (), kwargs = {'path': '/tmp/tmp-page2annexmnr_hi4x'}\ncwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\npwd_before = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\nexc_info = (<class 'TypeError'>, TypeError(\"test_AnnexRepo_commit() got multiple values for argument 'path'\"), <traceback object at 0x75ceb43db5a0>)\ncwd_after = '/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_b0cfdbb1b24a0a36fa48be6f7a2c68ed88d029a9'\n\n    @wraps(func)\n    def newfunc(*args, **kwargs):\n        cwd_before = os.getcwd()\n        pwd_before = getpwd()\n        exc_info = None\n        try:\n            func(*args, **kwargs)\n        except:\n            exc_info = sys.exc_info()\n        finally:\n            try:\n                cwd_after = os.getcwd()\n            except OSError as e:\n                lgr.warning(\"Failed to getcwd: %s\" % e)\n                cwd_after = None\n    \n        if cwd_after != cwd_before:\n            chpwd(pwd_before)\n            if not ok_to_chdir:\n                lgr.warning(\n                    \"%s changed cwd to %s. Mitigating and changing back to %s\"\n                    % (func, cwd_after, pwd_before))\n                # If there was already exception raised, we better reraise\n                # that one since it must be more important, so not masking it\n                # here with our assertion\n                if exc_info is None:\n                    assert_equal(cwd_before, cwd_after,\n                                 \"CWD changed from %s to %s\" % (cwd_before, cwd_after))\n    \n        if exc_info is not None:\n>           reraise(*exc_info)\n\ndatalad/tests/utils.py:1090: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n.venv/lib/python3.7/site-packages/six.py:724: in reraise\n    raise value\ndatalad/tests/utils.py:1066: in newfunc\n    func(*args, **kwargs)\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\narg = (), kw = {'path': '/tmp/tmp-page2annexmnr_hi4x'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_commit1fxu6clz'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_commit() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_AnnexRepo_add_to_annex __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annext9l2_55o'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_mkaywhe2', '/tmp/datalad_temp_clone_url_j60n0a_b', '/tmp/datalad_temp_clone_url_zoill_ew']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_clone_url_mkaywhe2'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_to_annex() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n__________________________ test_AnnexRepo_add_to_git ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annextwr1uv7t'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_1peeh6nd', '/tmp/datalad_temp_clone_url__2wkr5nc', '/tmp/datalad_temp_clone_url_2dstsngq']\nfake_dates = None, ntested = 1, uri = '/tmp/datalad_temp_clone_url_1peeh6nd'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_to_git() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n__________________ test_AnnexRepo_add_unexpected_direct_mode ___________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexfuc33be4'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_s0vw8vd1'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_s0vw8vd1'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_add_unexpected_direct_mode() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_____________________________ test_annex_backends ______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexdrebmocf'}\nfilename = '/tmp/datalad_temp_test_annex_backendszdzt9iie'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_backends() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n______________________________ test_annex_remove _______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex7ucvm342'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_p3ogye_g'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_p3ogye_g'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_annex_remove() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_________________________ test_annex_get_annexed_files _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexq1o1dgcy'}\ntkwargs_ = {'prefix': 'datalad_temp_tree_test_annex_get_annexed_files'}\nd = '/tmp/datalad_temp_tree_test_annex_get_annexed_filesq_ui9utl'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        tkwargs_ = get_tempfile_kwargs(tkwargs, prefix=\"tree\", wrapped=t)\n        d = tempfile.mkdtemp(**tkwargs_)\n        create_tree(d, tree, archives_leading_dir=archives_leading_dir)\n        try:\n>           return t(*(arg + (d,)), **kw)\nE           TypeError: test_annex_get_annexed_files() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:418: TypeError\n__________________________ test_annex_add_no_dotfiles __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexddmw1xfi'}\nfilename = '/tmp/datalad_temp_test_annex_add_no_dotfilesenlwbign'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_add_no_dotfiles() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________ test_annex_version_handling __________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexqgdprw9v'}\nfilename = '/tmp/datalad_temp_test_annex_version_handlingrnjwb5qg'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_annex_version_handling() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_____________________________ test_AnnexRepo_dirty _____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexvv1hwsjv'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_dirtycui56awz'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_dirty() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n________________________ test_AnnexRepo_set_remote_url _________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexwoo3udv3'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_set_remote_urlerosoeht'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_set_remote_url() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_________________________________ test_wanted __________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex3umpmn8s'}\nfilename = '/tmp/datalad_temp_test_wantedljsia6r0'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_wanted() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________________ test_AnnexRepo_metadata ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexcjyn8tqo'}\nfilename = '/tmp/datalad_temp_test_AnnexRepo_metadatavafjp41_'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_AnnexRepo_metadata() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________________ test_change_description ____________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexiv32_9vz'}\nfilename = '/tmp/datalad_temp_test_change_descriptionjq_48sre'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_change_description() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n___________________ test_AnnexRepo_get_corresponding_branch ____________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex753umi3g'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_qj_h4j4_'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_qj_h4j4_'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_corresponding_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n______________________ test_AnnexRepo_get_tracking_branch ______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex0qcz13ub'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_sxyxharc'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_sxyxharc'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_get_tracking_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n_______________________ test_AnnexRepo_is_managed_branch _______________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexbrngigyz'}, flavors_ = ['clone']\ntestrepos_uris = ['/tmp/datalad_temp_clone_url_ld26nf_j'], fake_dates = None\nntested = 1, uri = '/tmp/datalad_temp_clone_url_ld26nf_j'\n\n    @wraps(t)\n    @attr('with_testrepos')\n    def newfunc(*arg, **kw):\n        if on_windows:\n            raise SkipTest(\"Testrepo setup is broken on Windows\")\n    \n        # TODO: would need to either avoid this \"decorator\" approach for\n        # parametric tests or again aggregate failures like sweepargs does\n        flavors_ = _get_resolved_flavors(flavors)\n    \n        testrepos_uris = _get_testrepos_uris(regex, flavors_)\n        # we should always have at least one repo to test on, unless explicitly only\n        # network was requested by we are running without networked tests\n        if not (os.environ.get('DATALAD_TESTS_NONETWORK') and flavors == ['network']):\n            assert(testrepos_uris)\n        else:\n            if not testrepos_uris:\n                raise SkipTest(\"No non-networked repos to test on\")\n    \n        fake_dates = os.environ.get(\"DATALAD_FAKE__DATES\")\n        ntested = 0\n        for uri in testrepos_uris:\n            if count and ntested >= count:\n                break\n            ntested += 1\n            if __debug__:\n                lgr.debug('Running %s on %s' % (t.__name__, uri))\n            try:\n>               t(*(arg + (uri,)), **kw)\nE               TypeError: test_AnnexRepo_is_managed_branch() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:767: TypeError\n___________________________ test_fake_is_not_special ___________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexfcyjzr7i'}\nfilename = '/tmp/datalad_temp_test_fake_is_not_special5v90wmmu'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_fake_is_not_special() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_______________________________ test_fake_dates ________________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annex8er1koeh'}\nfilename = '/tmp/datalad_temp_test_fake_dates3hk8uv1f'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_fake_dates() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:591: TypeError\n_______________________ test_get_size_from_perc_complete _______________________\n\n    def test_get_size_from_perc_complete():\n        f = _get_size_from_perc_complete\n        eq_(f(0, 0), 0)\n        eq_(f(0, '0'), 0)\n        eq_(f(100, '0'), 0)  # we do not know better\n        eq_(f(1, '1'), 100)\n        # with no percentage info, we don't know better either:\n>       eq_(f(1, ''), 0)\n\nr2e_tests/test_1.py:2362: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\ncount = 1, perc = ''\n\n    def _get_size_from_perc_complete(count, perc):\n        \"\"\"A helper to get full size if know % and corresponding size\"\"\"\n>       portion = (float(perc) / 100.)\nE       ValueError: could not convert string to float:\n\ndatalad/support/annexrepo.py:3664: ValueError\n==================================== PASSES ====================================\n_____________________ test_ProcessAnnexProgressIndicators ______________________\n----------------------------- Captured stdout call -----------------------------\n\nfile1:   0%|          | 0.00/100 [00:00<?, ?B/s]\n                                                \n\nTotal:   0%|          | 0.00/100 [00:00<?, ?B/s]\nTotal (1 ok out of 2): 100%|██████████| 100/100 [00:00<00:00, 4.75kB/s]\n                                                                       \n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_ProcessAnnexProgressIndicators\nSKIPPED [2] .venv/lib/python3.7/site-packages/_pytest/unittest.py:371: TODO\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_file_has_content - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_file_has_content will be ignored\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_is_under_annex - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_is_under_annex will be ignored\nXFAIL r2e_tests/test_1.py::test_dropkey - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_dropkey will be ignored\nXFAIL r2e_tests/test_1.py::test_AnnexRepo_get_contentlocation - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_AnnexRepo_get_contentlocation will be ignored\nXFAIL r2e_tests/test_1.py::test_is_available - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_is_available will be ignored\nERROR r2e_tests/test_1.py::test_AnnexRepo_instance_from_clone\nERROR r2e_tests/test_1.py::test_AnnexRepo_crippled_filesystem\nERROR r2e_tests/test_1.py::test_AnnexRepo_set_direct_mode\nERROR r2e_tests/test_1.py::test_AnnexRepo_annex_proxy\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_file_key\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_outofspace\nERROR r2e_tests/test_1.py::test_AnnexRepo_web_remote\nERROR r2e_tests/test_1.py::test_AnnexRepo_migrating_backends\nERROR r2e_tests/test_1.py::test_AnnexRepo_backend_option\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_file_backend\nERROR r2e_tests/test_1.py::test_AnnexRepo_on_uninited_annex\nERROR r2e_tests/test_1.py::test_AnnexRepo_get\nERROR r2e_tests/test_1.py::test_AnnexRepo_addurl_to_file_batched\nERROR r2e_tests/test_1.py::test_annexrepo_fake_dates_disables_batched\nERROR r2e_tests/test_1.py::test_annex_ssh\nERROR r2e_tests/test_1.py::test_repo_version\nERROR r2e_tests/test_1.py::test_annex_copy_to\nERROR r2e_tests/test_1.py::test_annex_drop\nERROR r2e_tests/test_1.py::test_get_description\nERROR r2e_tests/test_1.py::test_AnnexRepo_flyweight\nERROR r2e_tests/test_1.py::test_AnnexRepo_get_toppath\nERROR r2e_tests/test_1.py::test_AnnexRepo_add_submodule\nERROR r2e_tests/test_1.py::test_AnnexRepo_status\nERROR r2e_tests/test_1.py::test_AnnexRepo_addurl_batched_and_set_metadata\nERROR r2e_tests/test_1.py::test_AnnexRepo_flyweight_monitoring_inode\nFAILED r2e_tests/test_1.py::test_AnnexRepo_instance_from_existing - datalad.s...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_instance_brand_new - TypeError: te...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_direct_mode - TypeError: test_A...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_direct_mode_gitrepo - TypeError...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_remote_na - TypeError: test_An...\nFAILED r2e_tests/test_1.py::test_find_batch_equivalence - TypeError: test_fin...\nFAILED r2e_tests/test_1.py::test_repo_info - TypeError: test_repo_info() got ...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_always_commit - TypeError: test_An...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_commit - TypeError: test_AnnexRepo...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_to_annex - TypeError: test_Ann...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_to_git - TypeError: test_Annex...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_add_unexpected_direct_mode - TypeE...\nFAILED r2e_tests/test_1.py::test_annex_backends - TypeError: test_annex_backe...\nFAILED r2e_tests/test_1.py::test_annex_remove - TypeError: test_annex_remove(...\nFAILED r2e_tests/test_1.py::test_annex_get_annexed_files - TypeError: test_an...\nFAILED r2e_tests/test_1.py::test_annex_add_no_dotfiles - TypeError: test_anne...\nFAILED r2e_tests/test_1.py::test_annex_version_handling - TypeError: test_ann...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_dirty - TypeError: test_AnnexRepo_...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_set_remote_url - TypeError: test_A...\nFAILED r2e_tests/test_1.py::test_wanted - TypeError: test_wanted() got multip...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_metadata - TypeError: test_AnnexRe...\nFAILED r2e_tests/test_1.py::test_change_description - TypeError: test_change_...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_corresponding_branch - TypeErr...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_get_tracking_branch - TypeError: t...\nFAILED r2e_tests/test_1.py::test_AnnexRepo_is_managed_branch - TypeError: tes...\nFAILED r2e_tests/test_1.py::test_fake_is_not_special - TypeError: test_fake_i...\nFAILED r2e_tests/test_1.py::test_fake_dates - TypeError: test_fake_dates() go...\nFAILED r2e_tests/test_1.py::test_get_size_from_perc_complete - ValueError: co...\n======== 28 failed, 1 passed, 2 skipped, 5 xfailed, 25 errors in 3.87s =========\n",
    "old_commit_res_stderr": ""
}
datalad/support/annexrepo.py datalad/support/tests/test_annexrepo.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `_get_size_from_perc_complete` crashes when percentage is missing

**Description:**
When the percentage value is not provided, the `_get_size_from_perc_complete` function fails instead of handling the missing input gracefully.

**Example Code:**
```python
result = _get_size_from_perc_complete(1, '')
```

**Expected Behavior:**
The function should return `0` when the percentage is not provided.

**Actual Behavior:**
A `ValueError` is raised because it attempts to convert an empty string to a float.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/annexrepo.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to git-annex by Joey Hess.

For further information on git-annex see https://git-annex.branchable.com/.

"""

import json
import logging
import math
import os
import re
import shlex
import tempfile
import time

from itertools import chain
from os import linesep
from os.path import curdir
from os.path import join as opj
from os.path import exists
from os.path import islink
from os.path import realpath
from os.path import lexists
from os.path import isdir
from os.path import isabs
from os.path import relpath
from os.path import normpath
from subprocess import Popen, PIPE
from multiprocessing import cpu_count
from weakref import WeakValueDictionary

from six import string_types, PY2
from six import iteritems
from six.moves import filter
from git import InvalidGitRepositoryError

from datalad import ssh_manager
from datalad.dochelpers import exc_str
from datalad.dochelpers import borrowdoc
from datalad.dochelpers import borrowkwargs
from datalad.utils import linux_distribution_name
from datalad.utils import nothing_cm
from datalad.utils import auto_repr
from datalad.utils import on_windows
from datalad.utils import swallow_logs
from datalad.utils import assure_list
from datalad.utils import _path_
from datalad.utils import generate_chunks
from datalad.utils import CMD_MAX_ARG
from datalad.utils import assure_unicode, assure_bytes
from datalad.utils import make_tempfile
from datalad.utils import unlink
from datalad.support.json_py import loads as json_loads
from datalad.cmd import GitRunner

# imports from same module:
from .repo import RepoInterface
from .gitrepo import GitRepo
from .gitrepo import NoSuchPathError
from .gitrepo import _normalize_path
from .gitrepo import normalize_path
from .gitrepo import normalize_paths
from .gitrepo import GitCommandError
from .gitrepo import to_options
from . import ansi_colors
from .external_versions import external_versions
from .exceptions import CommandNotAvailableError
from .exceptions import CommandError
from .exceptions import FileNotInAnnexError
from .exceptions import FileInGitError
from .exceptions import FileNotInRepositoryError
from .exceptions import AnnexBatchCommandError
from .exceptions import InsufficientArgumentsError
from .exceptions import OutOfSpaceError
from .exceptions import RemoteNotAvailableError
from .exceptions import OutdatedExternalDependency
from .exceptions import MissingExternalDependency
from .exceptions import IncompleteResultsError
from .exceptions import AccessDeniedError
from .exceptions import AccessFailedError

lgr = logging.getLogger('datalad.annex')

# Limit to # of CPUs and up to 8, but at least 3 to start with
N_AUTO_JOBS = min(8, max(3, cpu_count()))


class AnnexRepo(GitRepo, RepoInterface):
    """Representation of an git-annex repository.

    Paths given to any of the class methods will be interpreted as relative
    to PWD, in case this is currently beneath AnnexRepo's base dir
    (`self.path`). If PWD is outside of the repository, relative paths
    will be interpreted as relative to `self.path`. Absolute paths will be
    accepted either way.
    """

    # Begin Flyweight:
    _unique_instances = WeakValueDictionary()
    # End Flyweight:

    # Web remote has a hard-coded UUID we might (ab)use
    WEB_UUID = "00000000-0000-0000-0000-000000000001"

    # To be assigned and checked to be good enough upon first call to AnnexRepo
    # 6.20160923 -- --json-progress for get
    # 6.20161210 -- annex add  to add also changes (not only new files) to git
    # 6.20170220 -- annex status provides --ignore-submodules
    # 6.20180416 -- annex handles unicode filenames more uniformly
    # 6.20180913 -- annex fixes all known to us issues for v6
    GIT_ANNEX_MIN_VERSION = '6.20180913'
    git_annex_version = None

    # Class wide setting to allow insecure URLs. Used during testing, since
    # git annex 6.20180626 those will by default be not allowed for security
    # reasons
    _ALLOW_LOCAL_URLS = False

    def __init__(self, path, url=None, runner=None,
                 direct=None, backend=None, always_commit=True, create=True,
                 init=False, batch_size=None, version=None, description=None,
                 git_opts=None, annex_opts=None, annex_init_opts=None,
                 repo=None, fake_dates=False):
        """Creates representation of git-annex repository at `path`.

        AnnexRepo is initialized by giving a path to the annex.
        If no annex exists at that location, a new one is created.
        Optionally give url to clone from.

        Parameters
        ----------
        path: str
          Path to git-annex repository. In case it's not an absolute path, it's
          relative to PWD
        url: str, optional
          url to the to-be-cloned repository. Requires valid git url
          according to
          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .
        runner: Runner, optional
          Provide a Runner in case AnnexRepo shall not create it's own.
          This is especially needed in case of desired dry runs.
        direct: bool, optional
          If True, force git-annex to use direct mode
        backend: str, optional
          Set default backend used by this annex. This does NOT affect files,
          that are already annexed nor will it automatically migrate files,
          hat are 'getted' afterwards.
        create: bool, optional
          Create and initialize an annex repository at path, in case
          there is none. If set to False, and this repository is not an annex
          repository (initialized or not), an exception is raised.
        init: bool, optional
          Initialize git-annex repository (run "git annex init") if path is an
          annex repository which just was not yet initialized by annex (e.g. a
          fresh git clone). Note that if `create=True`, then initialization
          would happen
        batch_size: int, optional
          If specified and >0, instructs annex to batch this many commands before
          annex adds acts on git repository (e.g. adds them them to index for addurl).
        version: int, optional
          If given, pass as --version to `git annex init`
        description: str, optional
          Short description that humans can use to identify the
          repository/location, e.g. "Precious data on my laptop"
        """
        if self.git_annex_version is None:
            self._check_git_annex_version()

        # initialize
        self._uuid = None
        self._annex_common_options = []
        # Workaround for per-call config issue with git 2.11.0
        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False
        self.GIT_DIRECT_MODE_PROXY = False

        if annex_opts or annex_init_opts:
            lgr.warning("TODO: options passed to git-annex and/or "
                        "git-annex-init are currently ignored.\n"
                        "options received:\n"
                        "git-annex: %s\ngit-annex-init: %s" %
                        (annex_opts, annex_init_opts))

        fix_it = False
        try:
            super(AnnexRepo, self).__init__(path, url, runner=runner,
                                            create=create, repo=repo,
                                            git_opts=git_opts,
                                            fake_dates=fake_dates)
        except GitCommandError as e:
            if create and "Clone succeeded, but checkout failed." in str(e):
                lgr.warning("Experienced issues while cloning. "
                            "Trying to fix it, using git-annex-fsck.")
                fix_it = True
            else:
                raise e

        # Below while setting for direct mode workaround, we change
        # _GIT_COMMON_OPTIONS.
        # But we should not pass workarounds such as --worktree=.
        # -c core.bare=False to git annex commands, so for their
        # invocation we will keep and use pristine version of the
        # common options
        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]

        # check for possible SSH URLs of the remotes in order to set up
        # shared connections:
        for r in self.get_remotes():
            for url in [self.get_remote_url(r),
                        self.get_remote_url(r, push=True)]:
                if url is not None:
                    self._set_shared_connection(r, url)

        self.always_commit = always_commit

        if version is None:
            version = self.config.get("datalad.repo.version", None)
            # we might get an empty string here
            # TODO: if we use obtain() instead, we get an error complaining
            # '' cannot be converted to int (via Constraint as defined for
            # "datalad.repo.version" in common_cfg
            # => Allow conversion to result in None?
            if not version:
                version = None

        if fix_it:
            self._init(version=version, description=description)
            self.fsck()

        # Check whether an annex already exists at destination
        # XXX this doesn't work for a submodule!
        if not AnnexRepo.is_valid_repo(self.path):
            # so either it is not annex at all or just was not yet initialized
            if self.is_with_annex():
                # it is an annex repository which was not initialized yet
                if create or init:
                    lgr.debug('Annex repository was not yet initialized at %s.'
                              ' Initializing ...' % self.path)
                    self._init(version=version, description=description)
            elif create:
                lgr.debug('Initializing annex repository at %s...' % self.path)
                self._init(version=version, description=description)
            else:
                raise RuntimeError("No annex found at %s." % self.path)

        self._direct_mode = None  # we don't know yet

        # If we are in direct mode already, we need to make
        # this instance aware of that. This especially means, that we need to
        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().
        # Could happen in case we didn't specify anything, but annex forced
        # direct mode due to FS or an already existing repo was in direct mode,
        if self._is_direct_mode_from_config():
            self.set_direct_mode()

        # - only force direct mode; don't force indirect mode
        # - parameter `direct` has priority over config
        if direct is None:
            direct = (create or init) and \
                     self.config.getbool("datalad", "repo.direct", default=False)
        self._direct_mode = None  # we don't know yet
        if direct and not self.is_direct_mode():
            # direct mode is available below version 6 repos only.
            # Note: If 'annex.version' is missing in .git/config for some
            # reason, we need to try to set direct mode:
            repo_version = self.config.getint("annex", "version")
            if (repo_version is None) or (repo_version < 6):
                lgr.debug("Switching to direct mode (%s)." % self)
                self.set_direct_mode()
            else:
                # TODO: This may change to either not being a warning and/or
                # to use 'git annex unlock' instead.
                lgr.warning("direct mode not available for %s. Ignored." % self)

        self._batched = BatchedAnnexes(
            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)

        # set default backend for future annex commands:
        # TODO: Should the backend option of __init__() also migrate
        # the annex, in case there are annexed files already?
        if backend:
            self.set_default_backend(backend, persistent=True)

        if self._ALLOW_LOCAL_URLS:
            self._allow_local_urls()

    def _allow_local_urls(self):
        """Allow URL schemes and addresses which potentially could be harmful.

        For now it is internal method used within tests only
        """
        # from annex 6.20180626 file:/// and http://localhost access isn't
        # allowed by default
        self.config.add(
            'annex.security.allowed-url-schemes', 'http https file',
            'local')
        self.config.add(
            'annex.security.allowed-http-addresses', 'all',
            'local')

    def set_default_backend(self, backend, persistent=True, commit=True):
        """Set default backend

        Parameters
        ----------
        backend : str
        persistent : bool, optional
          If persistent, would add/commit to .gitattributes. If not -- would
          set within .git/config
        """
        # TODO: 'annex.backends' actually is a space separated list.
        # Figure out, whether we want to allow for a list here or what to
        # do, if there is sth in that setting already
        if persistent:
            # could be set in .gitattributes or $GIT_DIR/info/attributes
            if 'annex.backend' in self.get_gitattributes('.')['.']:
                lgr.debug(
                    "Not (re)setting backend since seems already set in git attributes"
                )
            else:
                lgr.debug("Setting annex backend to %s (persistently)", backend)
                self.config.set('annex.backends', backend, where='local')
                git_attributes_file = _path_(self.path, '.gitattributes')
                git_attributes = ''
                if exists(git_attributes_file):
                    with open(git_attributes_file) as f:
                        git_attributes = f.read()
                with open(git_attributes_file, 'a') as f:
                    if git_attributes and not git_attributes.endswith(os.linesep):
                        f.write(os.linesep)
                    f.write('* annex.backend=%s%s' % (backend, os.linesep))
                self.add(git_attributes_file, git=True)
                if commit:
                    self.commit(
                        "Set default backend for all files to be %s" % backend,
                        _datalad_msg=True,
                        files=[git_attributes_file]
                    )
        else:
            lgr.debug("Setting annex backend to %s (in .git/config)", backend)
            self.config.set('annex.backends', backend, where='local')

    def __del__(self):
        try:
            if hasattr(self, '_batched') and self._batched is not None:
                self._batched.close()
        except TypeError as e:
            # Workaround:
            # most likely something wasn't accessible anymore; doesn't really
            # matter since we wanted to delete it anyway.
            #
            # Nevertheless, in some cases might be an issue and it is a strange
            # thing to happen, since we check for things being None herein as
            # well as in super class __del__;
            # At least log it:
            lgr.debug(exc_str(e))
        try:
            super(AnnexRepo, self).__del__()
        except TypeError as e:
            # see above
            lgr.debug(exc_str(e))

    def _set_shared_connection(self, remote_name, url):
        """Make sure a remote with SSH URL uses shared connections.

        Set ssh options for annex on a per call basis, using
        '-c remote.<name>.annex-ssh-options'.

        Note
        ----
        There's currently no solution for using these connections, if the SSH
        URL is just connected to a file instead of a remote
        (`annex addurl` for example).

        Parameters
        ----------
        remote_name: str
        url: str
        """
        from datalad.support.network import is_ssh
        # Note:
        #
        # before any possible invocation of git-annex
        # Temporary approach to ssh connection sharing:
        # Register every ssh remote with the corresponding control master.
        # Issues:
        # - currently overwrites existing ssh config of the remote
        # - request SSHConnection instance and write config even if no
        #   connection needed (but: connection is not actually created/opened)
        # - no solution for a ssh url of a file (annex addurl)

        if is_ssh(url):
            c = ssh_manager.get_connection(url)
            ssh_cfg_var = "remote.{0}.annex-ssh-options".format(remote_name)
            # options to add:
            # Note: must use -S to overload -S provided by annex itself
            # if we provide -o ControlPath=... it is not in effect
            # Note: ctrl_path must not contain spaces, since it seems to be
            # impossible to anyhow guard them here
            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/
            cfg_string = "-o ControlMaster=auto -S %s" % c.ctrl_path
            # read user-defined options from .git/config:
            cfg_string_old = self.config.get(ssh_cfg_var, None)
            self._annex_common_options += \
                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'
                       ''.format(remote_name,
                                 (cfg_string_old + " ") if cfg_string_old else "",
                                 cfg_string
                                 )]

    def is_managed_branch(self, branch=None):
        """Whether `branch` is managed by git-annex.

        ATM this returns true in direct mode (branch 'annex/direct/my_branch')
        and if on an adjusted branch (annex v6 repository:
        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'

        Note: The term 'managed branch' is used to make clear it's meant to be
        more general than the v6 'adjusted branch'.

        Parameters
        ----------
        branch: str
          name of the branch; default: active branch

        Returns
        -------
        bool
          True if on a managed branch, False otherwise
        """

        if branch is None:
            branch = self.get_active_branch()
        # Note: `branch` might still be None, due to detached HEAD
        # (or no checkout at all)
        if branch and \
            (branch.startswith('annex/direct/') or
             branch.startswith('adjusted/')):
            return True
        return False

    def get_corresponding_branch(self, branch=None):
        """In case of a managed branch, get the corresponding one.

        If `branch` is not a managed branch, return that branch without any
        changes.

        Note: Since default for `branch` is the active branch,
        `get_corresponding_branch()` is equivalent to `get_active_branch()` if
        the active branch is not a managed branch.

        Parameters
        ----------
        branch: str
          name of the branch; defaults to active branch

        Returns
        -------
        str
          name of the corresponding branch if there is any, name of the queried
          branch otherwise.
        """

        if branch is None:
            branch = self.get_active_branch()

        if self.is_managed_branch(branch):
            if branch.startswith('annex/direct/'):
                cor_branch = branch[13:]
            elif branch.startswith('adjusted/'):
                if branch.endswith('(unlocked)'):
                    cor_branch = branch[9:-10]
                elif branch.endswith('(fixed)'):
                    cor_branch = branch[9:-7]
                else:
                    cor_branch = branch[9:]
                    lgr.warning("Unexpected naming of adjusted branch '{}'.{}"
                                "Assuming '{}' to be the corresponding branch."
                                "".format(branch, linesep, cor_branch))
            else:
                raise NotImplementedError(
                    "Detection of annex-managed branch '{}' follows a pattern "
                    "not implemented herein.".format(branch))
            return cor_branch

        else:
            return branch

    def get_tracking_branch(self, branch=None, corresponding=True):
        """Get the tracking branch for `branch` if there is any.

        By default returns the tracking branch of the corresponding branch if
        `branch` is a managed branch.

        Parameters
        ----------
        branch: str
          local branch to look up. If none is given, active branch is used.
        corresponding: bool
          If True actually look up the corresponding branch of `branch` (also if
          `branch` isn't explicitly given)

        Returns
        -------
        tuple
            (remote or None, refspec or None) of the tracking branch
        """

        if branch is None:
            branch = self.get_active_branch()

        return super(AnnexRepo, self).get_tracking_branch(
                        branch=self.get_corresponding_branch(branch)
                        if corresponding else branch)

    def _submodules_dirty_direct_mode(self,
            untracked=True, deleted=True, modified=True, added=True,
            type_changed=True, path=None):
        """Get modified submodules

        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/

        This is using git-annex-status with --ignore-submodules to not let
        git-status try to recurse into annex submodules without a working tree.
        Therefore we need to do the recursion on our own.

        Note, that added submodules will just be reported dirty. It's at very
        least difficult to distinguish whether a submodule in direct mode was
        just added or modified. ATM not worth the effort, I think.
        This is leads to a bit inconsistent reportings by AnnexRepo.status()
        whenever it needs to call this subroutine and there are added submodules.

        Intended to be used by AnnexRepo.status() internally.
        """

        # Note: We do a lazy recursion. The only thing we need to know is
        # whether or not a submodule is to be reported dirty. Once we already
        # know it is, there's no need to go any deeper in the hierarchy.
        # Apart from better performance, this also allows us to inspect each
        # submodule separately, and therefore be able to deal with mixed
        # hierarchies of git and annex submodules!

        modified_subs = []
        for sm in self.get_submodules():
            sm_dirty = False

            # First check for changes committed in the submodule, using
            # git submodule summary -- path,
            # since this can't be detected from within the submodule.
            if self.is_submodule_modified(sm.name):
                sm_dirty = True

            # check state of annex submodules, that might be in direct mode
            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),
                                         allow_noninitialized=False):

                sm_repo = AnnexRepo(opj(self.path, sm.path),
                                    create=False, init=False)

                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,
                                               modified=modified, added=added,
                                               type_changed=type_changed,
                                               submodules=False, path=path)
                if any([bool(sm_status[i]) for i in sm_status]):
                    sm_dirty = True

            # check state of submodule, that is a plain git or not an
            # initialized annex, which we can safely treat as a plain git, too.
            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):
                sm_repo = GitRepo(opj(self.path, sm.path))

                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters
                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,
                                    working_tree=deleted or modified or added or type_changed,
                                    untracked_files=untracked,
                                    submodules=False, path=path):
                    sm_dirty = True
            else:
                # uninitialized submodule
                # it can't be dirty and we can't recurse any deeper:
                continue

            if sm_dirty:
                # the submodule itself is dirty
                modified_subs.append(sm.path)
            else:
                # the submodule itself is clean, recurse:
                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.
                # we need get_status and this recursion method to be available
                # to both classes. Issue: We need to be able to come back to
                # AnnexRepo from GitRepo if there's again an annex beneath. But
                # we can't import AnnexRepo in gitrepo.py.
                modified_subs.extend(
                    sm_repo._submodules_dirty_direct_mode(
                        untracked=untracked, deleted=deleted,
                        modified=modified, added=added,
                        type_changed=type_changed, path=path
                    ))

        return modified_subs

    def get_status(self, untracked=True, deleted=True, modified=True, added=True,
                   type_changed=True, submodules=True, path=None):
        """Return various aspects of the status of the annex repository

        Note: Under certain circumstances newly added submodules might be
        reported as 'modified' rather tha 'added'.
        See `AnnexRepo._submodules_dirty_direct_mode` for details.

        Parameters
        ----------
        untracked
        deleted
        modified
        added
        type_changed
        submodules
        path

        Returns
        -------

        """

        self.precommit()

        options = assure_list(path) if path else []
        if not submodules:
            options.extend(to_options(ignore_submodules='all'))

        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)
        # internal call to 'git status' by 'git annex status' will fail
        # in submodules without a working tree (direct mode)
        # How to catch this case depends on annex version, since annex
        # exits zero until version 6.20170307

        def _fake_exception_wrapper(self, options_):
            """generate a faked `CommandError` from logged stderr output"""

            # this is for use with older annex, which didn't exit non-zero
            # in case of the failure we are interested in

            # TODO: If we are to keep this workaround (we probably rely on a
            # newer annex anyway), we should not use swallow_logs, since we
            # actually don't want to swallow it, but inspect it. Use a proper
            # handler/filter for the logger instead to not create temp files via
            # swallow_logs

            old_log_state = self.cmd_call_wrapper.log_outputs
            self.cmd_call_wrapper._log_opts['outputs'] = True

            with swallow_logs(new_level=logging.ERROR) as cml:
                # Note, that _run_annex_command_json returns a generator
                json_list = \
                    list(self._run_annex_command_json(
                        'status', opts=options_, expect_stderr=False))
            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state
            if "fatal:" in cml.out:
                raise CommandError(cmd="git annex status",
                                   msg=cml.out, stderr=cml.out)
            return json_list

        try:
            if self.git_annex_version < '6.20170307':
                json_list = _fake_exception_wrapper(self, options_=options)
            else:
                json_list = \
                    list(self._run_annex_command_json(
                        'status', opts=options, expect_stderr=False))
        except CommandError as e:
            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):
                lgr.debug("git-annex-status failed probably due to submodule in"
                          " direct mode. Trying to workaround.")
                # try again, ignoring submodules:
                options = [path] if path else []
                options.extend(to_options(ignore_submodules='all'))
                json_list = list(
                    self._run_annex_command_json('status', opts=options)
                )
                # separately get modified submodules:
                m_subs = \
                    self._submodules_dirty_direct_mode(untracked=untracked,
                                                       deleted=deleted,
                                                       modified=modified,
                                                       added=added,
                                                       type_changed=type_changed,
                                                       path=path)
                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)
            else:
                # not the known bug we want to catch
                raise e

        # END workaround

        key_mapping = [(untracked, 'untracked', '?'),
                       (deleted, 'deleted', 'D'),
                       (modified, 'modified', 'M'),
                       (added, 'added', 'A'),
                       (type_changed, 'type_changed', 'T')]
        from datalad.utils import with_pathsep
        return {key: [with_pathsep(i['file'])
                      if isdir(opj(self.path, i['file'])) else i['file']
                      # for consistency with 'git status' return directories
                      # with trailing path separator
                      for i in json_list if i['status'] == st]
                for cond, key, st in key_mapping if cond}

    @borrowdoc(GitRepo)
    def is_dirty(self, index=True, working_tree=False, untracked_files=True,
                 submodules=True, path=None):
        # TODO: Add doc on how this differs from GitRepo.is_dirty()
        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()

        if working_tree:
            # Note: annex repos don't always have a git working tree and the
            # behaviour in direct mode or V6 repos is fundamentally different
            # from that concept. There are no unstaged changes in direct mode
            # for example. Therefore the need to call this method with
            # 'working_tree=True' indicates invalid assumptions in the
            # calling code.

            # TODO: Better exception. InvalidArgumentError or sth ...
            raise CommandNotAvailableError(
                "Querying a git-annex repository for a clean/dirty "
                "working tree is an invalid concept.")
        # Again note, that 'annex status' isn't distinguishing staged and
        # unstaged changes, since this makes little sense for an annex repo
        # in general. Therefore we use only 'index' and 'untracked_files' to
        # specify what kind of dirtyness we are interested in:
        status = self.get_status(untracked=untracked_files, deleted=index,
                                 modified=index, added=index,
                                 type_changed=index, submodules=submodules,
                                 path=path)
        return any([bool(status[i]) for i in status])

    @property
    def untracked_files(self):
        """Get a list of untracked files
        """
        return self.get_status(untracked=True, deleted=False, modified=False,
                               added=False, type_changed=False, submodules=False,
                               path=None)['untracked']

    @classmethod
    def _check_git_annex_version(cls):
        ver = external_versions['cmd:annex']
        # in case it is missing
        if linux_distribution_name in {'debian', 'ubuntu'}:
            msg = "Install  git-annex-standalone  from NeuroDebian " \
                  "(http://neuro.debian.net)"
        else:
            msg = "Visit http://git-annex.branchable.com/install/"
        exc_kwargs = dict(
            name="git-annex",
            msg=msg,
            ver=cls.GIT_ANNEX_MIN_VERSION
        )
        if not ver:
            raise MissingExternalDependency(**exc_kwargs)
        elif ver < cls.GIT_ANNEX_MIN_VERSION:
            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)
        cls.git_annex_version = ver

    @staticmethod
    def get_size_from_key(key):
        """A little helper to obtain size encoded in a key"""
        if not key:
            return None
        try:
            size_str = key.split('-', 2)[1].lstrip('s')
        except IndexError:
            # has no 2nd field in the key
            return None
        return int(size_str) if size_str.isdigit() else None

    @normalize_path
    def get_file_size(self, path):
        fpath = opj(self.path, path)
        return 0 if not exists(fpath) else os.stat(fpath).st_size

    # TODO: Once the PR containing super class 'Repo' was merged, move there and
    # melt with GitRepo.get_toppath including tests for both
    @classmethod
    def get_toppath(cls, path, follow_up=True, git_options=None):
        """Return top-level of a repository given the path.

        Parameters
        -----------
        follow_up : bool
          If path has symlinks -- they get resolved by git.  If follow_up is
          True, we will follow original path up until we hit the same resolved
          path.  If no such path found, resolved one would be returned.
        git_options: list of str
          options to be passed to the git rev-parse call

        Return None if no parent directory contains a git repository.
        """

        # first try plain git result:
        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,
                                      git_options=git_options)
        if toppath == '':
            # didn't fail, so git itself didn't come to the conclusion
            # there is no repo, but we have no actual result;
            # might be an annex in direct mode
            if git_options is None:
                git_options = []
            # TODO: Apparently doesn't work with git 2.11.0
            # Note: Since we are in a classmethod, GitRepo.get_toppath uses
            # Runner directly instead of _git_custom_command, which is why the
            # common mechanics for direct mode are not applied.
            # This is why there is no solution for git 2.11 yet

            # Note 2: Actually, the above issue is irrelevant. The git
            # executable has no repository it is bound to, since it's the
            # purpose of the call to find this repository. Therefore
            # core.bare=False has no effect at all.

            # Disabled. See notes.
            # git_options.extend(['-c', 'core.bare=False'])
            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,
            #                               git_options=git_options)

            # basically a copy of code in GitRepo.get_toppath
            # except it uses 'git rev-parse --git-dir' as a workaround for
            # direct mode:

            from os.path import dirname
            from os import pardir

            cmd = ['git']
            if git_options:
                cmd.extend(git_options)

            cmd.append("rev-parse")
            if external_versions['cmd:git'] >= '2.13.0':
                cmd.append("--absolute-git-dir")
            else:
                cmd.append("--git-dir")

            try:
                toppath, err = GitRunner().run(
                    cmd,
                    cwd=path,
                    log_stdout=True, log_stderr=True,
                    expect_fail=True, expect_stderr=True)
                toppath = toppath.rstrip('\n\r')
            except CommandError:
                return None
            except OSError:
                toppath = AnnexRepo.get_toppath(dirname(path),
                                                follow_up=follow_up,
                                                git_options=git_options)

            if external_versions['cmd:git'] < '2.13.0':
                # we got a path relative to `path` instead of an absolute one
                toppath = opj(path, toppath)

            # we got the git-dir. Assuming the root dir we are looking for is
            # one level up:
            toppath = realpath(normpath(opj(toppath, pardir)))

            if follow_up:
                path_ = path
                path_prev = ""
                while path_ and path_ != path_prev:  # on top /.. = /
                    if realpath(path_) == toppath:
                        toppath = path_
                        break
                    path_prev = path_
                    path_ = dirname(path_)

        return toppath

    @classmethod
    def is_valid_repo(cls, path, allow_noninitialized=False):
        """Return True if given path points to an annex repository
        """
        # Note: default value for allow_noninitialized=False is important
        # for invalidating an instance via self._flyweight_invalid. If this is
        # changed, we also need to override _flyweight_invalid and explicitly
        # pass allow_noninitialized=False!

        def git_file_has_annex(p):
            """Return True if `p` contains a .git file, that points to a git
            dir with a subdir 'annex'"""
            _git = opj(p, '.git')
            if not os.path.isfile(_git):
                return False
            with open(_git, "r") as f:
                line = f.readline()
                if line.startswith("gitdir: "):
                    return exists(opj(p, line[8:], 'annex'))
                else:
                    lgr.debug("Invalid .git file: %s", _git)
                    return False

        initialized_annex = GitRepo.is_valid_repo(path) and \
            (exists(opj(path, '.git', 'annex')) or
             git_file_has_annex(path))

        if allow_noninitialized:
            try:
                return initialized_annex \
                    or GitRepo(path, create=False, init=False).is_with_annex()
            except (NoSuchPathError, InvalidGitRepositoryError):
                return False
        else:
            return initialized_annex

    def add_remote(self, name, url, options=None):
        """Overrides method from GitRepo in order to set
        remote.<name>.annex-ssh-options in case of a SSH remote."""
        super(AnnexRepo, self).add_remote(name, url, options if options else [])
        self._set_shared_connection(name, url)

    def set_remote_url(self, name, url, push=False):
        """Overrides method from GitRepo in order to set
        remote.<name>.annex-ssh-options in case of a SSH remote."""

        super(AnnexRepo, self).set_remote_url(name, url, push=push)
        self._set_shared_connection(name, url)

    def set_remote_dead(self, name):
        """Announce to annex that remote is "dead"
        """
        return self._annex_custom_command([], ["git", "annex", "dead", name])

    def is_remote_annex_ignored(self, remote):
        """Return True if remote is explicitly ignored"""
        return self.config.getbool(
            'remote.{}'.format(remote), 'annex-ignore',
            default=False
        )

    def is_special_annex_remote(self, remote, check_if_known=True):
        """Return either remote is a special annex remote

        Decides based on the presence of diagnostic annex- options
        for the remote
        """
        if check_if_known:
            if remote not in self.get_remotes():
                raise RemoteNotAvailableError(remote)
        sec = 'remote.{}'.format(remote)
        for opt in ('annex-externaltype', 'annex-webdav'):
            if self.config.has_option(sec, opt):
                return True
        return False

    @borrowkwargs(GitRepo)
    def get_remotes(self,
                    with_urls_only=False,
                    exclude_special_remotes=False):
        """Get known (special-) remotes of the repository

        Parameters
        ----------
        exclude_special_remotes: bool, optional
          if True, don't return annex special remotes

        Returns
        -------
        remotes : list of str
          List of names of the remotes
        """
        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)

        if exclude_special_remotes:
            return [
                remote for remote in remotes
                if not self.is_special_annex_remote(remote, check_if_known=False)
            ]
        else:
            return remotes

    def get_special_remotes(self):
        """Get info about all known (not just enabled) special remotes.

        Returns
        -------
        dict
          Keys are special remore UUIDs, values are dicts with arguments
          for `git-annex enableremote`. This includes at least the 'type'
          and 'name' of a special remote. Each type of special remote
          may require addition arguments that will be available in the
          respective dictionary.
        """
        try:
            stdout, stderr = self._git_custom_command(
                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],
                expect_fail=True)
        except CommandError as e:
            if 'Not a valid object name git-annex:remote.log' in e.stderr:
                # no special remotes configures
                return {}
            else:
                # some unforseen error
                raise e
        argspec = re.compile(r'^([^=]*)=(.*)$')
        srs = {}
        for line in stdout.splitlines():
            # be precise and split by spaces
            fields = line.split(' ')
            # special remote UUID
            sr_id = fields[0]
            # the rest are config args for enableremote
            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])
            srs[sr_id] = sr_info
        return srs

    def __repr__(self):
        return "<AnnexRepo path=%s (%s)>" % (self.path, type(self))

    def _run_annex_command(self, annex_cmd,
                           git_options=None, annex_options=None,
                           backend=None, jobs=None,
                           files=None,
                           **kwargs):
        """Helper to run actual git-annex calls

        Unifies annex command calls.

        Parameters
        ----------
        annex_cmd: str
            the actual git-annex command, like 'init' or 'add'
        git_options: list of str
            options to be passed to git
        annex_options: list of str
            options to be passed to the git-annex command
        backend: str
            backend to be used by this command; Currently this can also be
            achieved by having an item '--backend=XXX' in annex_options.
            This may change.
        jobs : int
        files: list, optional
            If command passes list of files
        **kwargs
            these are passed as additional kwargs to datalad.cmd.Runner.run()

        Raises
        ------
        CommandNotAvailableError
            if an annex command call returns "unknown command"
        """
        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []
        backend = ['--backend=%s' % backend] if backend else []

        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS
        annex_options = annex_options[:] if annex_options else []
        if self._annex_common_options:
            annex_options = self._annex_common_options + annex_options

        if not self.always_commit:
            git_options += ['-c', 'annex.alwayscommit=false']

        if git_options:
            cmd_list = ['git'] + git_options + ['annex']
        else:
            cmd_list = ['git-annex']
        if jobs:
            annex_options += ['-J%d' % jobs]

        cmd_list += [annex_cmd] + backend + debug + annex_options

        env = kwargs.pop("env", None)
        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        if files:
            cmd_list += ['--'] + files
        try:
            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)
        except CommandError as e:
            if e.stderr and "git-annex: Unknown command '%s'" % annex_cmd in e.stderr:
                raise CommandNotAvailableError(str(cmd_list),
                                               "Unknown command:"
                                               " 'git-annex %s'" % annex_cmd,
                                               e.code, e.stdout, e.stderr)
            else:
                raise e

    def _run_simple_annex_command(self, *args, **kwargs):
        """Run an annex command and return its output, of which expect 1 line

        Just a little helper to interact with basic annex commands and process
        their output while ignoring some messages

        Parameters
        ----------
        **kwargs: all passed into _run
        """
        out, err = self._run_annex_command(
            *args, **kwargs
        )
        lines = out.rstrip('\n').splitlines()
        # ignore some lines which might appear on a fresh clone
        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/
        lines_ = [
            l for l in lines
            if not re.search(
                '\((merging .* into git-annex|recording state ).*\.\.\.\)', l
            )
        ]
        assert(len(lines_) <= 1)
        return lines_[0] if lines_ else None

    def _is_direct_mode_from_config(self):
        """Figure out if in direct mode from the git config.

        Since relies on reading config, expensive to be used often

        Returns
        -------
        True if in direct mode, False otherwise.
        """
        # If .git/config lacks an entry "direct",
        # it's actually indirect mode.
        self.config.reload()
        return self.config.getbool("annex", "direct", False)

    def is_direct_mode(self):
        """Return True if annex is in direct mode

        Returns
        -------
        True if in direct mode, False otherwise.
        """
        # TEMP: Disable lazy loading and make sure to read from file every time
        # instead, since we might have several instances pointing to the very
        # same repo atm. TODO: We can remove that, right?
        self.repo.config_reader()._is_initialized = False
        self.repo.config_reader().read()
        self._direct_mode = None

        if self._direct_mode is None:
            # we need to figure it out
            self._direct_mode = self._is_direct_mode_from_config()
        return self._direct_mode

    def is_crippled_fs(self):
        """Return True if git-annex considers current filesystem 'crippled'.

        Returns
        -------
        True if on crippled filesystem, False otherwise
        """

        self.config.reload()
        return self.config.getbool("annex", "crippledfilesystem", False)

    def set_direct_mode(self, enable_direct_mode=True):
        """Switch to direct or indirect mode

        Parameters
        ----------
        enable_direct_mode: bool
            True means switch to direct mode,
            False switches to indirect mode

        Raises
        ------
        CommandNotAvailableError
            in case you try to switch to indirect mode on a crippled filesystem
        """
        if self.is_crippled_fs() and not enable_direct_mode:
            raise CommandNotAvailableError(
                cmd="git-annex indirect",
                msg="Can't switch to indirect mode on that filesystem.")

        self._run_annex_command('direct' if enable_direct_mode else 'indirect',
                                expect_stderr=True)
        self.config.reload()

        # For paranoid we will just re-request
        self._direct_mode = None
        assert(self.is_direct_mode() == enable_direct_mode)

        if self.is_direct_mode():
            # adjust git options for plain git calls on this repo:
            # Note: Not sure yet, whether this solves the issue entirely or we
            # still need 'annex proxy' in some cases ...

            lgr.debug("detected git version: %s" % external_versions['cmd:git'])

            if external_versions['cmd:git'] >= '2.9.0':
                # workaround for git 2.9.0, which for some reason ignores the
                # per-call config "-c core.bare=False", but respects the value
                # if it is set in .git/config
                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True

            # TEMP: nevertheless use this option to inject it into gitpython
            # TODO: Solve it and change to "elif"
            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:
                # standard direct mode procedure part I:
                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])
            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:
                # standard direct mode procedure part II:
                self._GIT_COMMON_OPTIONS.append('--work-tree=.')

    def _git_custom_command(self, *args, **kwargs):

        if self.GIT_DIRECT_MODE_PROXY:
            proxy_str = "git annex proxy -- "
            proxy_list = ['git', 'annex', 'proxy', '--']
            cmd = kwargs.pop("cmd_str", None)
            if not cmd:
                cmd = args[1]
            assert(cmd is not None)

            if isinstance(cmd, string_types):
                cmd = proxy_str + cmd
            else:
                cmd = proxy_list + cmd

            args = (args[0], cmd) + args[2:]
            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:
            old = self.config.get('core.bare')
            lgr.debug("old config: %s(%s)" % (old, type(old)))
            if old is not False:
                self.config.set('core.bare', 'False', where='local')

            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

            if old is None:
                self.config.unset('core.bare', where='local')
            elif old:
                self.config.set('core.bare', old, where='local')
            return out, err

        else:
            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

    def _init(self, version=None, description=None):
        """Initializes an annex repository.

        Note: This is intended for private use in this class by now.
        If you have an object of this class already,
        there shouldn't be a need to 'init' again.

        """
        # MIH: this function is required for re-initing repos. The logic
        # in the constructor is rather convoluted and doesn't acknowledge
        # the case of a perfectly healthy annex that just needs a new
        # description
        # will keep leading underscore in the name for know, but this is
        # not private
        # TODO: provide git and git-annex options.
        # TODO: Document (or implement respectively) behaviour in special cases
        # like direct mode (if it's different), not existing paths, etc.
        opts = []
        if description is not None:
            opts += [description]
        if version is not None:
            opts += ['--version', '{0}'.format(version)]
        if not len(opts):
            opts = None

        # Note: git-annex-init kills a possible tracking branch for
        # 'annex/direct/my_branch', if we just cloned from a repo in direct
        # mode. We want to preserve the information about the tracking branch,
        # as if the source repo wasn't in direct mode.
        # Note 2: Actually we do it for all 'managed branches'. This might turn
        # out to not be necessary
        sections_to_preserve = ["branch.{}".format(branch)
                                for branch in self.get_branches()
                                if self.is_managed_branch(branch)
                                and "branch.{}".format(branch) in
                                self.config.sections()]
        for sct in sections_to_preserve:
            orig_branch = sct[7:]
            new_branch = self.get_corresponding_branch(orig_branch)
            new_section = "branch.{}".format(new_branch)
            for opt in self.config.options(sct):
                orig_value = self.config.get_value(sct, opt)
                new_value = orig_value.replace(orig_branch, new_branch)
                self.config.add(var=new_section + "." + opt,
                                value=new_value,
                                where='local',
                                reload=False)
        self._run_annex_command(
            'init',
            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,
            annex_options=opts)
        # TODO: When to expect stderr?
        # on crippled filesystem for example (think so)?
        self.config.reload()

    @normalize_paths
    def get(self, files, remote=None, options=None, jobs=None, key=False):
        """Get the actual content of files

        Parameters
        ----------
        files : list of str
            paths to get
        remote : str, optional
            from which remote to fetch content
        options : list of str, optional
            commandline options for the git annex get command
        jobs : int or None, optional
            how many jobs to run in parallel (passed to git-annex call).
            If not specified (None), then
        key : bool, optional
            If provided file value is actually a key

        Returns
        -------
        files : list of dict
        """
        options = options[:] if options else []

        if remote:
            if remote not in self.get_remotes():
                raise RemoteNotAvailableError(
                    remote=remote,
                    cmd="get",
                    msg="Remote is not known. Known are: %s"
                    % (self.get_remotes(),)
                )
            options += ['--from', remote]

        # analyze provided files to decide which actually are needed to be
        # fetched

        if not key:
            expected_downloads, fetch_files = self._get_expected_files(
                files, ['--not', '--in', 'here'])
        else:
            fetch_files = files
            assert len(files) == 1, "When key=True only a single file be provided"
            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}

        if not fetch_files:
            lgr.debug("No files found needing fetching.")
            return []

        if len(fetch_files) != len(files):
            lgr.debug("Actually getting %d files", len(fetch_files))

        # TODO: provide more meaningful message (possibly aggregating 'note'
        #  from annex failed ones
        # TODO: reproduce DK's bug on OSX, and either switch to
        #  --batch mode (I don't think we have --progress support in long
        #  alive batch processes ATM),
        if key:
            kwargs = {'opts': options + ['--key'] + files}
        else:
            kwargs = {'opts': options, 'files': files}
        results = self._run_annex_command_json(
            'get',
            # TODO: eventually make use of --batch mode
            jobs=jobs,
            expected_entries=expected_downloads,
            progress=True,
            **kwargs
        )
        results_list = list(results)
        # TODO:  should we here compare fetch_files against result_list
        # and vomit an exception of incomplete download????
        return results_list

    def _get_expected_files(self, files, expr):
        """Given a list of files, figure out what to be downloaded

        Parameters
        ----------
        files
        expr: list
          Expression to be passed into annex's find

        Returns
        -------
        expected_files : dict
          key -> size
        fetch_files : list
          files to be fetched
        """
        lgr.debug("Determine what files match the query to work with")
        # Let's figure out first which files/keys and of what size to download
        expected_files = {}
        fetch_files = []
        keys_seen = set()
        unknown_sizes = []  # unused atm
        # for now just record total size, and
        for j in self._run_annex_command_json(
                'find', opts=expr, files=files
        ):
            # TODO: some files might not even be here.  So in current fancy
            # output reporting scheme we should then theoretically handle
            # those cases here and say 'impossible' or something like that
            if not j.get('success', True):
                # TODO: I guess do something with yielding and filtering for
                # what need to be done and what not
                continue
            key = j['key']
            size = j.get('bytesize')
            if key in keys_seen:
                # multiple files could point to the same key.  no need to
                # request multiple times
                continue
            keys_seen.add(key)
            assert j['file']
            fetch_files.append(j['file'])
            if size and size.isdigit():
                expected_files[key] = int(size)
            else:
                expected_files[key] = None
                unknown_sizes.append(j['file'])
        return expected_files, fetch_files

    @normalize_paths
    def add(self, files, git=None, backend=None, options=None, jobs=None,
            git_options=None, annex_options=None, update=False):
        """Add file(s) to the repository.

        Parameters
        ----------
        files: list of str
          list of paths to add to the annex
        git: bool
          if True, add to git instead of annex.
        backend:
        options:
        update: bool
          --update option for git-add. From git's manpage:
           Update the index just where it already has an entry matching
           <pathspec>. This removes as well as modifies index entries to match
           the working tree, but adds no new files.

           If no <pathspec> is given when --update option is used, all tracked
           files in the entire working tree are updated (old versions of Git
           used to limit the update to the current directory and its
           subdirectories).

           Note: Used only, if a call to git-add instead of git-annex-add is
           performed

        Returns
        -------
        list of dict
        """

        return list(self.add_(
            files, git=git, backend=backend, options=options, jobs=jobs,
            git_options=git_options, annex_options=annex_options, update=update
        ))

    def add_(self, files, git=None, backend=None, options=None, jobs=None,
            git_options=None, annex_options=None, update=False):
        """Like `add`, but returns a generator"""
        if update and not git:
            raise InsufficientArgumentsError("option 'update' requires 'git', too")

        if git_options:
            # TODO: note that below we would use 'add with --dry-run
            # so passed here options might need to be passed into it??
            lgr.warning("add: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []
        # Note: As long as we support direct mode, one should not call
        # super().add() directly. Once direct mode is gone, we might remove
        # `git` parameter and call GitRepo's add() instead.

        def _get_to_be_added_recs(paths):
            """Try to collect what actually is going to be added

            This is used for progress information
            """

            if self.is_direct_mode():
                # we already know we can't use --dry-run
                for r in self._process_git_get_output(
                        linesep.join(["'{}'".format(p.encode('utf-8'))
                        for p in paths])):
                    yield r
                    return
            else:
                # Note: if a path involves a submodule in direct mode, while we
                # are not in direct mode at current level, we might still fail.
                # Hence the except clause is still needed. However, this is
                # unlikely, since direct mode usually should be used only, if it
                # was enforced by FS and/or OS and therefore concerns the entire
                # hierarchy.
                _git_options = ['--dry-run', '-N', '--ignore-missing']
                try:
                    for r in super(AnnexRepo, self).add_(
                            files, git_options=_git_options, update=update):
                        yield r
                        return
                except CommandError as e:
                    if AnnexRepo._is_annex_work_tree_message(e.stderr):
                        lgr.warning(
                            "Known bug in direct mode."
                            "We can't use --dry-run when there are submodules in "
                            "direct mode, because the internal call to git status "
                            "fails. To be resolved by using (Dataset's) status "
                            "instead of a git-add --dry-run altogether.")
                        # fake the return for now
                        for r in self._process_git_get_output(
                                linesep.join(["'{}'".format(f)
                                for f in files])):
                            yield r
                            return
                    else:
                        # unexpected failure
                        raise e

        # Theoretically we could have done for git as well, if it could have
        # been batched
        # Call git annex add for any to have full control of either to go
        # to git or to annex
        # 1. Figure out what actually will be added
        to_be_added_recs = _get_to_be_added_recs(files)
        # collect their sizes for the progressbar
        expected_additions = {
            rec['file']: self.get_file_size(rec['file'])
            for rec in to_be_added_recs
        }

        # if None -- leave it to annex to decide
        if git is not None:
            options += [
                '-c',
                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])
            ]
            if git:
                # to maintain behaviour similar to git
                options += ['--include-dotfiles']

        if git and update:
            # explicitly use git-add with --update instead of git-annex-add
            # TODO: This might still need some work, when --update AND files
            # are specified!
            if self.is_direct_mode() and not files:
                self.GIT_DIRECT_MODE_PROXY = True
            try:
                for r in super(AnnexRepo, self).add(
                        files,
                        git=True,
                        git_options=git_options,
                        update=update):
                    yield r
            finally:
                if self.is_direct_mode() and not files:
                    # don't accidentally cause other git calls to be done
                    # via annex-proxy
                    self.GIT_DIRECT_MODE_PROXY = False

        else:
            for r in self._run_annex_command_json(
                    'add',
                    opts=options,
                    files=files,
                    backend=backend,
                    expect_fail=True,
                    jobs=jobs,
                    expected_entries=expected_additions,
                    expect_stderr=True):
                yield r

    def proxy(self, git_cmd, **kwargs):
        """Use git-annex as a proxy to git

        This is needed in case we are in direct mode, since there's no git
        working tree, that git can handle.

        Parameters
        ----------
        git_cmd: list of str
            the actual git command
        `**kwargs`: dict, optional
            passed to _run_annex_command

        Returns
        -------
        (stdout, stderr)
            output of the command call
        """
        # TODO: We probably don't need it anymore

        if not self.is_direct_mode():
            lgr.warning("proxy() called in indirect mode: %s" % git_cmd)
            raise CommandNotAvailableError(cmd="git annex proxy",
                                           msg="Proxy doesn't make sense"
                                               " if not in direct mode.")
        return self._run_annex_command('proxy',
                                       annex_options=['--'] + git_cmd,
                                       **kwargs)

    @normalize_paths
    def get_file_key(self, files):
        """Get key of an annexed file.

        Parameters
        ----------
        files: str or list
            file(s) to look up

        Returns
        -------
        str or list
            keys used by git-annex for each of the files;
            in case of a list an empty string is returned if there was no key
            for that file
        """

        if len(files) > 1:
            return self._batched.get('lookupkey', path=self.path)(files)
        else:
            files = files[0]
            # single file
            # keep current implementation
            # TODO: This should change, but involves more RF'ing and an
            # alternative regarding FileNotInAnnexError
            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages

            try:
                out, err = self._run_annex_command(
                    'lookupkey',
                    files=[files],
                    expect_fail=True
                )
            except CommandError as e:
                if e.code == 1:
                    if not exists(opj(self.path, files)):
                        raise IOError(e.code, "File not found.", files)
                    # XXX you don't like me because I can be real slow!
                    elif files in self.get_indexed_files():
                        # if we got here, the file is present and in git,
                        # but not in the annex
                        raise FileInGitError(cmd=cmd_str,
                                             msg="File not in annex, but git: %s"
                                                 % files,
                                             filename=files)
                    else:
                        raise FileNotInAnnexError(cmd=cmd_str,
                                                  msg="File not in annex: %s"
                                                      % files,
                                                  filename=files)
                else:
                    # Not sure, whether or not this can actually happen
                    raise e

            entries = out.rstrip(linesep).splitlines()
            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated
            entries = list(filter(lambda x: not x.startswith('('), entries))
            if len(entries) > 1:
                lgr.warning("Got multiple entries in reply asking for a key of a file: %s"
                            % (str(entries)))
            elif not entries:
                raise FileNotInAnnexError("Could not get a key for a file(s) %s -- empty output" % files)
            return entries[0]

    @normalize_paths
    def lock(self, files, options=None):
        """undo unlock

        Use  this to undo an unlock command if you don't want to modify the
        files any longer, or have made modifications you want to discard.

        Parameters
        ----------
        files: list of str
        options: list of str
        """

        options = options[:] if options else []
        self._run_annex_command('lock', annex_options=options, files=files)
        # note: there seems to be no output by annex if success.

    @normalize_paths
    def unlock(self, files, options=None):
        """unlock files for modification

        Parameters
        ----------
        files: list of str
        options: list of str

        Returns
        -------
        list of str
          successfully unlocked files
        """

        options = options[:] if options else []

        if self.is_direct_mode():

            # TODO:
            # If anything there should be a CommandNotAvailableError now:
            lgr.debug("'%s' is in direct mode, "
                      "'annex unlock' not available", self)
            lgr.warning("In direct mode there is no 'unlock'. However if "
                        "the file's content is present, it is kind of "
                        "unlocked. Therefore just checking whether this is "
                        "the case.")
            # TODO/FIXME:
            # Note: the following isn't exactly nice, if `files` is a dir.
            # For a "correct" result we would need to report all files within
            # potential dir(s) in `files`, that are annexed and have content.
            # Also note, that even now files in git might be reported "unlocked",
            # since they have content. This might be a confusing result.
            # On the other hand, this is solved on the level of Dataset.unlock
            # by annotating those paths 'notneeded' beforehand.
            return [f for f in files if self.file_has_content(f)]

        else:

            # TODO: catch and parse output if failed (missing content ...)
            std_out, std_err = \
                self._run_annex_command(
                    'unlock', annex_options=options, files=files
                )

            return [line.split()[1]
                    for line in std_out.splitlines()
                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']

    def adjust(self, options=None):
        """enter an adjusted branch

        This command is only available in a v6 git-annex repository.

        Parameters
        ----------
        options: list of str
          currently requires '--unlock' or '--fix';
          default: --unlock
        """
        # TODO: Do we want to catch the case that
        # "adjusted/<current_branch_name>(unlocked)" already exists and
        # just check it out? Or fail like annex itself does?

        # version check:
        if not self.config.get("annex.version") == '6':
            raise CommandNotAvailableError(cmd='git annex adjust',
                                           msg='git-annex-adjust requires a '
                                               'version 6 repository')

        options = options[:] if options else to_options(unlock=True)
        self._run_annex_command('adjust', annex_options=options)

    @normalize_paths
    def unannex(self, files, options=None):
        """undo accidental add command

        Use this to undo an accidental git annex add command. Note that for
        safety, the content of the file remains in the annex, until you use git
        annex unused and git annex dropunused.

        Parameters
        ----------
        files: list of str
        options: list of str

        Returns
        -------
        list of str
          successfully unannexed files
        """

        options = options[:] if options else []

        std_out, std_err = self._run_annex_command(
            'unannex', annex_options=options, files=files
        )
        return [line.split()[1] for line in std_out.splitlines()
                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']

    @normalize_paths(map_filenames_back=True)
    def find(self, files, batch=False):
        """Run `git annex find` on file(s).

        Parameters
        ----------
        files: list of str
            files to find under annex
        batch: bool, optional
            initiate or continue with a batched run of annex find, instead of just
            calling a single git annex find command. If any items in `files`
            are directories, this value is treated as False.

        Returns
        -------
        A dictionary the maps each item in `files` to its `git annex find`
        result. Items without a successful result will be an empty string, and
        multi-item results (which can occur for if `files` includes a
        directory) will be returned as a list.
        """
        objects = {}
        # Ignore batch=True if any path is a directory because `git annex find
        # --batch` always returns an empty string for directories.
        if batch and not any(isdir(opj(self.path, f)) for f in files):
            find = self._batched.get('find', json=True, path=self.path)
            objects = {f: json_out.get("file", "")
                       for f, json_out in zip(files, find(files))}
        else:
            for f in files:
                try:
                    obj, er = self._run_annex_command(
                        'find', files=[f],
                        annex_options=["--print0"],
                        expect_fail=True
                    )
                    items = obj.rstrip("\0").split("\0")
                    objects[f] = items[0] if len(items) == 1 else items
                except CommandError:
                    objects[f] = ''

        return objects

    def _check_files(self, fn, quick_fn, files, allow_quick, batch):
        # Helper that isolates the common logic in `file_has_content` and
        # `is_under_annex`. `fn` is the annex command used to do the check, and
        # `quick_fn` is the non-annex variant.
        is_v6 = self.config.get("annex.version") == "6"
        if is_v6 or self.is_direct_mode() or batch or not allow_quick:
            # We're only concerned about modified files in V6 mode. In V5
            # `find` returns an empty string for unlocked files, and in direct
            # mode everything looks modified, so we don't even bother.
            modified = self.get_changed_files() if is_v6 else []
            annex_res = fn(files, normalize_paths=False, batch=batch)
            return [bool(annex_res.get(f) and
                         not (is_v6 and normpath(f) in modified))
                    for f in files]
        else:  # ad-hoc check which should be faster than call into annex
            return [quick_fn(f) for f in files]

    @normalize_paths
    def file_has_content(self, files, allow_quick=True, batch=False):
        """Check whether files have their content present under annex.

        Parameters
        ----------
        files: list of str
            file(s) to check for being actually present.
        allow_quick: bool, optional
            allow quick check, based on having a symlink into .git/annex/objects.
            Works only in non-direct mode (TODO: thin mode)

        Returns
        -------
        list of bool
            For each input file states either file has content locally
        """
        # TODO: Also provide option to look for key instead of path

        def quick_check(filename):
            filepath = opj(self.path, filename)
            if islink(filepath):                    # if symlink
                # find abspath of node pointed to by symlink
                target_path = realpath(filepath)  # realpath OK
                # TODO: checks for being not outside of this repository
                # Note: ben removed '.git/' from '.git/annex/objects',
                # since it is not true for submodules, whose '.git' is a
                # symlink and being resolved to some
                # '.git/modules/.../annex/objects'
                return exists(target_path) and 'annex/objects' in str(target_path)
            return False

        return self._check_files(self.find, quick_check,
                                 files, allow_quick, batch)

    @normalize_paths
    def is_under_annex(self, files, allow_quick=True, batch=False):
        """Check whether files are under annex control

        Parameters
        ----------
        files: list of str
            file(s) to check for being under annex
        allow_quick: bool, optional
            allow quick check, based on having a symlink into .git/annex/objects.
            Works only in non-direct mode (TODO: thin mode)

        Returns
        -------
        list of bool
            For each input file states either file is under annex
        """
        # theoretically in direct mode files without content would also be
        # broken symlinks on the FSs which support it, but that would complicate
        # the matters

        # This is an ugly hack to prevent files from being treated as
        # remotes by `git annex info`. See annex's `nameToUUID'`.
        files = [opj(curdir, f) for f in files]

        def check(files, **kwargs):
            # Filter out directories because it doesn't make sense to ask if
            # they are under annex control and `info` can only handle
            # non-directories.
            return self.info([f for f in files if not isdir(f)],
                             fast=True, **kwargs)

        def quick_check(filename):
            filepath = opj(self.path, filename)
            # todo checks for being not outside of this repository
            # Note: ben removed '.git/' from '.git/annex/objects',
            # since it is not true for submodules, whose '.git' is a
            # symlink and being resolved to some
            # '.git/modules/.../annex/objects'
            return (
                islink(filepath)
                and 'annex/objects' in str(realpath(filepath))  # realpath OK
            )

        return self._check_files(check, quick_check,
                                 files, allow_quick, batch)

    def init_remote(self, name, options):
        """Creates a new special remote

        Parameters
        ----------
        name: str
            name of the special remote
        """
        # TODO: figure out consistent way for passing options + document

        self._run_annex_command('initremote', annex_options=[name] + options)
        self.config.reload()

    def enable_remote(self, name, env=None):
        """Enables use of an existing special remote

        Parameters
        ----------
        name: str
            name, the special remote was created with
        """

        try:
            self._run_annex_command(
                'enableremote',
                annex_options=[name],
                expect_fail=True,
                log_stderr=True,
                env=env)
        except CommandError as e:
            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):
                raise AccessDeniedError(e.stderr)
            elif 'FailedConnectionException' in e.stderr:
                raise AccessFailedError(e.stderr)
            else:
                raise e
        self.config.reload()

    def merge_annex(self, remote=None):
        """Merge git-annex branch

        Merely calls `sync` with the appropriate arguments.

        Parameters
        ----------
        remote: str, optional
          Name of a remote to be "merged".
        """
        self.sync(
            remotes=remote, push=False, pull=False, commit=False, content=False,
            all=False)

    def sync(self, remotes=None, push=True, pull=True, commit=True,
             content=False, all=False, fast=False):
        """Synchronize local repository with remotes

        Use  this  command  when you want to synchronize the local repository
        with one or more of its remotes. You can specify the remotes (or
        remote groups) to sync with by name; the default if none are specified
        is to sync with all remotes.

        Parameters
        ----------
        remotes: str, list(str), optional
          Name of one or more remotes to be sync'ed.
        push : bool
          By default, git pushes to remotes.
        pull : bool
          By default, git pulls from remotes
        commit : bool
          A commit is done by default. Disable to avoid  committing local
          changes.
        content : bool
          Normally, syncing does not transfer the contents of annexed
          files.  This option causes the content of files in the work tree
          to also be uploaded and downloaded as necessary.
        all : bool
          This option, when combined with `content`, makes all available
          versions of all files be synced, when preferred content settings
          allow
        fast : bool
          Only sync with the remotes with the lowest annex-cost value
          configured
        """
        # In direct mode annex-sync fails, if commit=True
        # apparently sync is calling git status internally, which then fails
        # in the submodule. (As we already know)
        # stdout:
        # commit  add second ok
        # (recording state in git...)
        #
        # failed
        # (recording state in git...)
        #
        # stderr:
        # fatal: This operation must be run in a work tree
        # fatal: 'git status --porcelain' failed in submodule submod
        # git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","-f"] exited 123)
        # fatal: This operation must be run in a work tree
        # fatal: 'git status --porcelain' failed in submodule submod
        # git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","-f"] exited 123)

        # TODO: Workaround

        args = []
        args.extend(to_options(push=push, no_push=not push,
                               # means: '--push' if push else '--no-push'
                               pull=pull, no_pull=not pull,
                               commit=commit, no_commit=not commit,
                               content=content, no_content=not content,
                               all=all,
                               fast=fast))
        args.extend(assure_list(remotes))
        self._run_annex_command('sync', annex_options=args)

    @normalize_path
    def add_url_to_file(self, file_, url, options=None, backend=None,
                        batch=False, git_options=None, annex_options=None,
                        unlink_existing=False):
        """Add file from url to the annex.

        Downloads `file` from `url` and add it to the annex.
        If annex knows `file` already,
        records that it can be downloaded from `url`.

        Note: Consider using the higher-level `download_url` instead.

        Parameters
        ----------
        file_: str

        url: str

        options: list
            options to the annex command

        batch: bool, optional
            initiate or continue with a batched run of annex addurl, instead of just
            calling a single git annex addurl command

        unlink_existing: bool, optional
            by default crashes if file already exists and is under git.
            With this flag set to True would first remove it.

        Returns
        -------
        dict
          In batch mode only ATM returns dict representation of json output returned
          by annex
        """

        if git_options:
            lgr.warning("add_url_to_file: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []
        git_options = []
        kwargs = dict(backend=backend)
        if lexists(opj(self.path, file_)) and \
                unlink_existing and \
                not self.is_under_annex(file_):
            # already under git, we can't addurl for under annex
            lgr.warning(
                "File %s:%s is already under git, removing so it could possibly"
                " be added under annex", self, file_
            )
            unlink(opj(self.path, file_))
        if not batch or self.fake_dates_enabled:
            if batch:
                lgr.debug("Not batching addurl call "
                          "because fake dates are enabled")
            files_opt = '--file=%s' % file_
            if PY2:
                files_opt = assure_bytes(files_opt)
            out_json = self._run_annex_command_json(
                'addurl',
                opts=options + [files_opt] + [url],
                progress=True,
                expected_entries={file_: None},
                **kwargs
            )
            if len(out_json) != 1:
                raise AssertionError(
                    "should always be a single-item list, Got: %s"
                    % str(out_json))
            # Make the output's structure match bcmd's.
            out_json = out_json[0]
            # Don't capture stderr, since download progress provided by wget uses
            # stderr.
        else:
            options += ['--with-files']
            if backend:
                options += ['--backend=%s' % backend]
            # Initializes (if necessary) and obtains the batch process
            bcmd = self._batched.get(
                # Since backend will be critical for non-existing files
                'addurl_to_file_backend:%s' % backend,
                annex_cmd='addurl',
                git_options=git_options,
                annex_options=options,  # --raw ?
                path=self.path,
                json=True
            )
            try:
                out_json = bcmd((url, file_))
            except Exception as exc:
                # if isinstance(exc, IOError):
                #     raise
                raise AnnexBatchCommandError(
                    cmd="addurl",
                    msg="Adding url %s to file %s failed due to %s" % (url, file_, exc_str(exc)))
            assert(out_json['command'] == 'addurl')
        if not out_json.get('success', False):
            raise (AnnexBatchCommandError if batch else CommandError)(
                    cmd="addurl",
                    msg="Error, annex reported failure for addurl (url='%s'): %s"
                    % (url, str(out_json)))
        return out_json

    def add_urls(self, urls, options=None, backend=None, cwd=None,
                 jobs=None,
                 git_options=None, annex_options=None):
        """Downloads each url to its own file, which is added to the annex.

        Parameters
        ----------
        urls: list of str

        options: list, optional
            options to the annex command

        cwd: string, optional
            working directory from within which to invoke git-annex
        """

        if git_options:
            lgr.warning("add_urls: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []

        self._run_annex_command('addurl', annex_options=options + urls,
                                backend=backend, log_online=True,
                                log_stderr=False, cwd=cwd)
        # Don't capture stderr, since download progress provided by wget uses
        # stderr.

        # currently simulating similar return value, assuming success
        # for all files:
        # TODO: Make return values consistent across both *Repo classes!
        return [{u'file': f, u'success': True} for f in urls]

    @normalize_path
    def rm_url(self, file_, url):
        """Record that the file is no longer available at the url.

        Parameters
        ----------
        file_: str

        url: str
        """

        self._run_annex_command('rmurl', files=[file_, url])

    @normalize_path
    def get_urls(self, file_, key=False, batch=False):
        """Get URLs for a file/key

        Parameters
        ----------
        file_: str
        key: bool, optional
            Either provided files are actually annex keys
        """
        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']

    @normalize_paths
    def drop(self, files, options=None, key=False, jobs=None):
        """Drops the content of annexed files from this repository.

        Drops only if possible with respect to required minimal number of
        available copies.

        Parameters
        ----------
        files: list of str
            paths to drop
        options : list of str, optional
            commandline options for the git annex drop command
        jobs : int, optional
            how many jobs to run in parallel (passed to git-annex call)

        Returns
        -------
        list(JSON objects)
          'success' item in each object indicates failure/success per file
          path.
        """

        # annex drop takes either files or options
        # --all, --unused, --key, or --incomplete
        # for now, most simple test; to be replaced by a more general solution
        # (exception thrown by _run_annex_command)
        if not files and \
                (not options or
                 not any([o in options for o in
                          ["--all", "--unused", "--key", "--incomplete"]])):
            raise InsufficientArgumentsError("drop() requires at least to "
                                             "specify 'files' or 'options'")

        options = assure_list(options)

        if key:
            # we can't drop multiple in 1 line, and there is no --batch yet, so
            # one at a time
            files = assure_list(files)
            options = options + ['--key']
            res = [
                self._run_annex_command_json(
                    'drop',
                    opts=options + [k],
                    jobs=jobs)
                for k in files
            ]
            # `normalize_paths` ... magic, useful?
            if len(files) == 1:
                return res[0]
            else:
                return res
        else:
            return self._run_annex_command_json(
                'drop',
                opts=options,
                files=files,
                jobs=jobs)

    def drop_key(self, keys, options=None, batch=False):
        """Drops the content of annexed files from this repository referenced by keys

        Dangerous: it drops without checking for required minimal number of
        available copies.

        Parameters
        ----------
        keys: list of str, str

        batch: bool, optional
            initiate or continue with a batched run of annex dropkey, instead of just
            calling a single git annex dropkey command
        """
        keys = [keys] if isinstance(keys, string_types) else keys

        options = options[:] if options else []
        options += ['--force']
        if not batch or self.fake_dates_enabled:
            if batch:
                lgr.debug("Not batching drop_key call "
                          "because fake dates are enabled")
            json_objects = self._run_annex_command_json(
                'dropkey', opts=options, files=keys, expect_stderr=True
            )
        else:
            json_objects = self._batched.get(
                'dropkey',
                annex_options=options, json=True, path=self.path
            )(keys)
        for j in json_objects:
            assert j.get('success', True)

    # TODO: a dedicated unit-test
    def _whereis_json_to_dict(self, j):
        """Convert json record returned by annex whereis --json to our dict representation for it
        """
        assert (j.get('success', True) is True)
        # process 'whereis' containing list of remotes
        remotes = {remote['uuid']: {x: remote.get(x, None)
                                    for x in ('description', 'here', 'urls')
                                    }
                   for remote in j.get('whereis')}
        if self.WEB_UUID in remotes:
            assert(remotes[self.WEB_UUID]['description'] == 'web')
        return remotes

    def _run_annex_command_json(self, command,
                                opts=None,
                                jobs=None,
                                files=None,
                                expected_entries=None,
                                progress=False,
                                **kwargs):
        """Run an annex command with --json and load output results into a tuple of dicts

        Parameters
        ----------
        expected_entries : dict, optional
          If provided `filename/key: size` dictionary, will be used to create
          ProcessAnnexProgressIndicators to display progress
        progress: bool, optional
          Whether to request/handle --json-progress
        """
        progress_indicators = None
        try:
            if expected_entries:
                progress_indicators = ProcessAnnexProgressIndicators(
                    expected=expected_entries
                )
                kwargs = kwargs.copy()
                kwargs.update(dict(
                    log_stdout=progress_indicators,
                    log_stderr='offline',  # False, # to avoid lock down
                    log_online=True
                ))
            # TODO: refactor to account for possible --batch ones
            annex_options = ['--json']
            if progress:
                annex_options += ['--json-progress']

            if jobs == 'auto':
                jobs = N_AUTO_JOBS
            if jobs and jobs != 1:
                annex_options += ['-J%d' % jobs]
            if opts:
                # opts might be the '--key' which should go last
                annex_options += opts

            # TODO: RF to use --batch where possible instead of splitting
            # into multiple invocations
            if not files:
                file_chunks = [[]]
            else:
                files = assure_list(files)
                maxl = max(map(len, files))
                chunk_size = max(
                    1,  # should at least be 1. If blows then - not our fault
                    (CMD_MAX_ARG
                     - len(command)
                     - sum((len(x) + 3) for x in annex_options)
                     - 4   # for '--' below
                     - 50  # for safety since we are to add more options
                    // (maxl + 3)  # +3 for possible quotes and a space
                    )
                )
                file_chunks = generate_chunks(files, chunk_size)
            out, err = "", ""
            for file_chunk in file_chunks:
                out_, err_ = self._run_annex_command(
                    command,
                    annex_options=annex_options + ['--'] + file_chunk,
                    **kwargs)
                out += out_
                err += err_
        except CommandError as e:
            # Note: A call might result in several 'failures', that can be or
            # cannot be handled here. Detection of something, we can deal with,
            # doesn't mean there's nothing else to deal with.

            # OutOfSpaceError:
            # Note:
            # doesn't depend on anything in stdout. Therefore check this before
            # dealing with stdout
            out_of_space_re = re.search(
                "not enough free space, need (.*) more", e.stderr
            )
            if out_of_space_re:
                raise OutOfSpaceError(cmd="annex %s" % command,
                                      sizemore_msg=out_of_space_re.groups()[0])

            # RemoteNotAvailableError:
            remote_na_re = re.search(
                "there is no available git remote named \"(.*)\"", e.stderr
            )
            if remote_na_re:
                raise RemoteNotAvailableError(cmd="annex %s" % command,
                                              remote=remote_na_re.groups()[0])

            # TEMP: Workaround for git-annex bug, where it reports success=True
            # for annex add, while simultaneously complaining, that it is in
            # a submodule:
            # TODO: For now just reraise. But independently on this bug, it
            # makes sense to have an exception for that case
            in_subm_re = re.search(
                "fatal: Pathspec '(.*)' is in submodule '(.*)'", e.stderr
            )
            if in_subm_re:
                raise e

            # Note: try to approach the covering of potential annex failures
            # in a more general way:
            # first check stdout:
            if all([line.startswith('{') and line.endswith('}')
                    for line in e.stdout.splitlines()]):
                # we have the usual json output on stdout. Therefore we can
                # probably return and don't need to raise; so get stdout
                # for json loading:
                out = e.stdout
            else:
                out = None

            # Note: Workaround for not existing files as long as annex doesn't
            # report it within JSON response:
            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/
            not_existing = [
                line.split()[1] for line in e.stderr.splitlines()
                if line.startswith('git-annex:') and
                   line.endswith('not found')
            ]
            if not_existing:
                if out is None:
                    # we create the error reporting herein. If all files were
                    # not found, there is nothing on stdout and we don't need
                    # anything
                    out = ""
                if not out.endswith(linesep):
                    out += linesep
                out += linesep.join(
                    ['{{"command": "{cmd}", "file": "{path}", '
                     '"note": "{note}",'
                     '"success":false}}'.format(
                         cmd=command, path=f, note="not found")
                     for f in not_existing])

            # Note: insert additional code here to analyse failure and possibly
            # raise a custom exception

            # if we didn't raise before, just depend on whether or not we seem
            # to have some json to return. It should contain information on
            # failure in keys 'success' and 'note'
            # TODO: This is not entirely true. 'annex status' may return empty,
            # while there was a 'fatal:...' in stderr, which should be a
            # failure/exception
            # Or if we had empty stdout but there was stderr
            if out is None or (not out and e.stderr):
                raise e
            if e.stderr:
                # else just warn about present errors
                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x
                lgr.warning(
                    "Running %s resulted in stderr output: %s",
                    command, shorten(e.stderr)
                )
        finally:
            if progress_indicators:
                progress_indicators.finish()

        json_objects = (json_loads(line)
                        for line in out.splitlines() if line.startswith('{'))
        # protect against progress leakage
        json_objects = [j for j in json_objects if 'byte-progress' not in j]
        return json_objects

    # TODO: reconsider having any magic at all and maybe just return a list/dict always
    @normalize_paths
    def whereis(self, files, output='uuids', key=False, options=None, batch=False):
        """Lists repositories that have actual content of file(s).

        Parameters
        ----------
        files: list of str
            files to look for
        output: {'descriptions', 'uuids', 'full'}, optional
            If 'descriptions', a list of remotes descriptions returned is per
            each file. If 'full', for each file a dictionary of all fields
            is returned as returned by annex
        key: bool, optional
            Either provided files are actually annex keys
        options: list, optional
            Options to pass into git-annex call

        Returns
        -------
        list of list of unicode  or dict
            if output == 'descriptions', contains a list of descriptions of remotes
            for each input file, describing the remote for each remote, which
            was found by git-annex whereis, like::

                u'me@mycomputer:~/where/my/repo/is [origin]' or
                u'web' or
                u'me@mycomputer:~/some/other/clone'

            if output == 'uuids', returns a list of uuids.
            if output == 'full', returns a dictionary with filenames as keys
            and values a detailed record, e.g.::

                {'00000000-0000-0000-0000-000000000001': {
                  'description': 'web',
                  'here': False,
                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']
                }}
        """
        if batch:
            lgr.warning("TODO: --batch mode for whereis.  Operating serially")

        OUTPUTS = {'descriptions', 'uuids', 'full'}
        if output not in OUTPUTS:
            raise ValueError(
                "Unknown value output=%r. Known are %s"
                % (output, ', '.join(map(repr, OUTPUTS)))
            )

        options = assure_list(options, copy=True)
        if key:
            kwargs = {'opts': options + ["--key"] + files}
        else:
            kwargs = {'files': files}

        json_objects = self._run_annex_command_json('whereis', **kwargs)
        if output in {'descriptions', 'uuids'}:
            return [
                [remote.get(output[:-1]) for remote in j.get('whereis')]
                if j.get('success') else []
                for j in json_objects
            ]
        elif output == 'full':
            # TODO: we might want to optimize storage since many remotes entries will be the
            # same so we could just reuse them instead of brewing copies
            return {
                j['key' if (key or '--all' in options) else 'file']
                : self._whereis_json_to_dict(j)
                for j in json_objects
                if not j.get('key').endswith('.this-is-a-test-key')
            }

    # TODO:
    # I think we should make interface cleaner and less ambigious for those annex
    # commands which could operate on globs, files, and entire repositories, separating
    # those out, e.g. annex_info_repo, annex_info_files at least.
    # If we make our calling wrappers work without relying on invoking from repo topdir,
    # then returned filenames would not need to be mapped, so we could easily work on dirs
    # and globs.
    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?
    @normalize_paths(map_filenames_back=True)
    def info(self, files, batch=False, fast=False):
        """Provide annex info for file(s).

        Parameters
        ----------
        files: list of str
            files to look for

        Returns
        -------
        dict
          Info for each file
        """

        options = ['--bytes', '--fast'] if fast else ['--bytes']

        if not batch:
            json_objects = self._run_annex_command_json(
                'info', opts=options, files=files)
        else:
            json_objects = self._batched.get(
                'info',
                annex_options=options, json=True, path=self.path
            )(files)

        # Some aggressive checks. ATM info can be requested only per file
        # json_objects is a generator, let's keep it that way
        # assert(len(json_objects) == len(files))
        # and that they all have 'file' equal to the passed one
        out = {}
        for j, f in zip(json_objects, files):
            assert(j.pop('file') == f)
            if not j['success']:
                j = None
            else:
                assert(j.pop('success') is True)
                # convert size to int
                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None
                # and pop the "command" field
                j.pop("command")
            out[f] = j
        return out

    def repo_info(self, fast=False):
        """Provide annex info for the entire repository.

        Returns
        -------
        dict
          Info for the repository, with keys matching the ones returned by annex
        """

        options = ['--bytes', '--fast'] if fast else ['--bytes']

        json_records = list(self._run_annex_command_json('info', opts=options))
        assert(len(json_records) == 1)

        # TODO: we need to abstract/centralize conversion from annex fields
        # For now just tune up few for immediate usability
        info = json_records[0]
        for k in info:
            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):
                size = info[k].split()[0]
                if size.isdigit():
                    info[k] = int(size)
                else:
                    lgr.debug("Size %r reported to be %s, setting to None", k, size)
                    info[k] = None
        assert(info.pop('success'))
        assert(info.pop('command') == 'info')
        return info  # just as is for now

    def get_annexed_files(self, with_content_only=False, patterns=None):
        """Get a list of files in annex

        Parameters
        ----------
        with_content_only : bool, optional
            Only list files whose content is present.
        patterns : list, optional
            Globs to pass to annex's `--include=`. Files that match any of
            these will be returned (i.e., they'll be separated by `--or`).

        Returns
        -------
        A list of file names
        """
        if not patterns:
            args = [] if with_content_only else ['--include', "*"]
        else:
            if len(patterns) == 1:
                args = ['--include', patterns[0]]
            else:
                args = ['-(']
                for pat in patterns[:-1]:
                    args.extend(['--include', pat, "--or"])
                args.extend(['--include', patterns[-1]])
                args.append('-)')

            if with_content_only:
                args.extend(['--in', 'here'])
        out, err = self._run_annex_command('find', annex_options=args)
        # TODO: JSON
        return out.splitlines()

    def get_preferred_content(self, property, remote=None):
        """Get preferred content configuration of a repository or remote

        Parameters
        ----------
        property : {'wanted', 'required', 'group'}
          Type of property to query
        remote : str, optional
          If not specified (None), returns the property for the local
          repository.

        Returns
        -------
        str
          Either the setting is returned, or an empty string if there
          is none.

        Raises
        ------
        ValueError
          If an unknown property label is given.

        CommandError
          If the annex call errors.
        """
        if property not in ('wanted', 'required', 'group'):
            raise ValueError(
                'unknown preferred content property: {}'.format(property))
        return self._run_simple_annex_command(
            property,
            annex_options=[remote or '.'])

    def set_preferred_content(self, property, expr, remote=None):
        """Set preferred content configuration of a repository or remote

        Parameters
        ----------
        property : {'wanted', 'required', 'group'}
          Type of property to query
        expr : str
          Any expression or label supported by git-annex for the
          given property.
        remote : str, optional
          If not specified (None), sets the property for the local
          repository.

        Returns
        -------
        str
          Raw git-annex output in response to the set command.

        Raises
        ------
        ValueError
          If an unknown property label is given.

        CommandError
          If the annex call errors.
        """
        if property not in ('wanted', 'required', 'group'):
            raise ValueError(
                'unknown preferred content property: {}'.format(property))
        return self._run_simple_annex_command(
            property,
            annex_options=[remote or '.', expr])

    def get_groupwanted(self, name):
        """Get `groupwanted` expression for a group `name`

        Parameters
        ----------
        name : str
           Name of the groupwanted group
        """
        return self._run_simple_annex_command(
            'groupwanted', annex_options=[name]
        )

    def set_groupwanted(self, name, expr):
        """Set `expr` for the `name` groupwanted"""
        return self._run_simple_annex_command(
            'groupwanted', annex_options=[name, expr]
        )

    def precommit(self):
        """Perform pre-commit maintenance tasks, such as closing all batched annexes
        since they might still need to flush their changes into index
        """
        if self._batched is not None:
            self._batched.close()
        super(AnnexRepo, self).precommit()

    @borrowdoc(GitRepo)
    def commit(self, msg=None, options=None, _datalad_msg=False,
               careless=True, files=None, proxy=False):
        self.precommit()

        if files:
            files = assure_list(files)

        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy
        #       in direct mode. This is needed in very special cases, which
        #       might go away once we figured out a better way. In any case, it
        #       should turn into something that is automatically considered and
        #       not done by the caller of this method.

        if proxy:
            if not self.is_direct_mode():
                raise CommandNotAvailableError(
                    cmd="git-annex-proxy",
                    msg="git-annex-proxy is available in direct mode only")
            else:
                if _datalad_msg:
                    msg = self._get_prefixed_commit_msg(msg)
                if not msg:
                    if options:
                        if "--allow-empty-message" not in options:
                            options.append("--allow-empty-message")
                    else:
                        options = ["--allow-empty-message"]

                # committing explicitly given paths in direct mode via proxy
                # used to fail, because absolute paths are used. Using annex
                # proxy this leads to an error (path outside repository)
                if files:
                    if options is None:
                        options = []
                    for i in range(len(files)):
                        if isabs(files[i]):
                            options.append(normpath(relpath(files[i],
                                                            start=self.path)))
                        else:
                            options.append(files[i])
                try:
                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +
                               (options if options else []),
                               expect_stderr=True, expect_fail=True)
                except CommandError as e:
                    if 'nothing to commit' in e.stdout:
                        if careless:
                            lgr.debug("nothing to commit in {}. "
                                      "Ignored.".format(self))
                        else:
                            raise
                    elif 'no changes added to commit' in e.stdout or \
                            'nothing added to commit' in e.stdout:
                        if careless:
                            lgr.debug("no changes added to commit in {}. "
                                      "Ignored.".format(self))
                        else:
                            raise
                    elif "did not match any file(s) known to git." in e.stderr:
                        # TODO:
                        # Improve FileNotInXXXXError classes to better deal with
                        # multiple files; Also consider PathOutsideRepositoryError
                        raise FileNotInRepositoryError(cmd=e.cmd,
                                                       msg="File(s) unknown to git",
                                                       code=e.code,
                                                       filename=linesep.join(
                                                    [l for l in e.stderr.splitlines()
                                                     if l.startswith("pathspec")]))
                    else:
                        raise
        else:

            # Note: See the note on `proxy` parameter at the top of this method.
            #       Trying to automatically use git-annex-proxy, whenever we
            #       fail to commit the usual way via options to git in direct
            #       mode. In particular this can happen if sth was staged via
            #       git-annex-proxy, which is needed for --update option for
            #       example.

            if files:
                # Raise FileNotInRepositoryError if `files` aren't tracked.
                try:
                    super(AnnexRepo, self).commit(
                        "dryrun", options=["--dry-run", "--no-status"],
                        files=files)
                except CommandError as e:
                    if not (self.is_direct_mode() and
                            AnnexRepo._is_annex_work_tree_message(e.stderr)):
                        raise
                    # The git call may fail with
                    #   fatal: this operation must be run in a work tree
                    #   fatal: 'git status --porcelain=2' failed in submodule ...
                    # But that error message doesn't matter for the purpose of
                    # the "file is tracked" check. It won't be shown if there
                    # is a pathspec error.
                    lgr.debug("Ignoring commit --dry-run failure")
            try:
                alt_index_file = None

                direct_mode = self.is_direct_mode()
                # we might need to avoid explicit paths
                files_to_commit = None if direct_mode else files
                if direct_mode and files:
                    # In direct mode, if we commit file(s) they would get
                    # committed directly into git ignoring possibly being
                    # staged by annex.  So, if not all files are committed, and
                    # assuming that what is to be committed is staged (!could be
                    # wrong assumption), we need to prepare a custom index, and
                    # commit without specifying any paths
                    changed_files = {
                        staged: set(self.get_changed_files(staged=staged))
                        for staged in (True,) #  False}
                    }
                    files_set = {
                        _normalize_path(self.path, f) if isabs(f) else f
                        for f in files}
                    files_notstaged = files_set.difference(changed_files[True])
                    # Lazy .add('.') results in ALL (performance?) files listed
                    # here even if they are not changed at all and could be ignored
                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])
                    # if files_changed_notstaged:
                    #     # To be safe(r), let's just blow for now!
                    #     # We could be smart(er) and do all the git or annex add
                    #     # but let's see if we run into those cases first
                    #     import pdb; pdb.set_trace()
                    #     raise RuntimeError(
                    #         "To commit files in direct mode, please first git "
                    #         "or git-annex add them first!  Files: %s"
                    #         % ', '.join(files_changed_notstaged)
                    #     )
                    # Files which were staged but not among files
                    staged_not_to_commit = changed_files[True].difference(files_set)
                    if staged_not_to_commit:
                        # Need an alternative index_file
                        with make_tempfile() as index_file:
                            alt_index_file = index_file
                            index_tree = self.repo.git.write_tree()
                            self.repo.git.read_tree(index_tree,
                                                    index_output=index_file)
                            # Reset the files we are not to be committed
                            self._git_custom_command(
                                list(staged_not_to_commit),
                                ['git', 'reset'],
                                index_file=alt_index_file)

                            super(AnnexRepo, self).commit(
                                msg, options,
                                _datalad_msg=_datalad_msg,
                                careless=careless,
                                index_file=alt_index_file)

                    # in any case we will not specify files explicitly
                    files_to_commit = None
                if not alt_index_file:
                    super(AnnexRepo, self).commit(msg, options,
                                                  _datalad_msg=_datalad_msg,
                                                  careless=careless,
                                                  files=files_to_commit)
            except CommandError as e:
                # TODO: just remove this???
                if self.is_direct_mode() and \
                    AnnexRepo._is_annex_work_tree_message(e.stderr):
                    lgr.debug("Commit failed. "
                              "Trying to commit via git-annex-proxy.")
                    self.commit(msg, options, _datalad_msg=_datalad_msg,
                                careless=careless, files=files, proxy=True)
                else:
                    raise
            finally:
                if alt_index_file and os.path.exists(alt_index_file):
                    unlink(alt_index_file)

    @normalize_paths(match_return_type=False)
    def remove(self, files, force=False, **kwargs):
        """Remove files from git/annex (works in direct mode as well)

        Parameters
        ----------
        files
        force: bool, optional
        """

        # TODO: parameter 'force' unnecessary => kwargs / to_options
        self.precommit()  # since might interfere

        return super(AnnexRepo, self).remove(files, force=force,
                                             normalize_paths=False,
                                             **kwargs)

    def get_contentlocation(self, key, batch=False):
        """Get location of the key content

        Normally under .git/annex objects in indirect mode and within file
        tree in direct mode.

        Unfortunately there is no (easy) way to discriminate situations
        when given key is simply incorrect (not known to annex) or its content
        not currently present -- in both cases annex just silently exits with -1


        Parameters
        ----------
        key: str
            key
        batch: bool, optional
            initiate or continue with a batched run of annex contentlocation

        Returns
        -------
        str
            path relative to the top directory of the repository. If no content
            is present, empty string is returned
        """

        if not batch:
            try:
                out, err = self._run_annex_command('contentlocation',
                                                   annex_options=[key],
                                                   expect_fail=True)
                return out.rstrip(linesep).splitlines()[0]
            except CommandError:
                return ''
        else:
            return self._batched.get('contentlocation', path=self.path)(key)

    @normalize_paths(serialize=True)
    def is_available(self, file_, remote=None, key=False, batch=False):
        """Check if file or key is available (from a remote)

        In case if key or remote is misspecified, it wouldn't fail but just keep
        returning False, although possibly also complaining out loud ;)

        Parameters
        ----------
        file_: str
            Filename or a key
        remote: str, optional
            Remote which to check.  If None, possibly multiple remotes are checked
            before positive result is reported
        key: bool, optional
            Either provided files are actually annex keys
        batch: bool, optional
            Initiate or continue with a batched run of annex checkpresentkey

        Returns
        -------
        bool
            with True indicating that file/key is available from (the) remote
        """

        if key:
            key_ = file_
        else:
            key_ = self.get_file_key(file_)  # ?, batch=batch

        annex_input = (key_,) if not remote else (key_, remote)

        if not batch:
            try:
                out, err = self._run_annex_command('checkpresentkey',
                                                   annex_options=list(annex_input),
                                                   expect_fail=True)
                assert(not out)
                return True
            except CommandError:
                return False
        else:
            annex_cmd = ["checkpresentkey"] + ([remote] if remote else [])
            out = self._batched.get(
                ':'.join(annex_cmd), annex_cmd,
                path=self.path)(key_)
            try:
                return {
                    # happens on travis in direct/heavy-debug mode, that process
                    # exits and closes stdout (upon unknown key) before we could
                    # read it, so we get None as the stdout.
                    # see https://github.com/datalad/datalad/issues/2330
                    # but it is associated with an unknown key, and for consistency
                    # we report False there too, as to ''
                    None: False,
                    '': False,  # when remote is misspecified ... stderr carries the msg
                    '0': False,
                    '1': True,
                }[out]
            except KeyError:
                raise ValueError(
                    "Received output %r from annex, whenever expect 0 or 1" % out
                )

    @normalize_paths(match_return_type=False)
    def _annex_custom_command(
            self, files, cmd_str, log_stdout=True, log_stderr=True,
            log_online=False, expect_stderr=False, cwd=None, env=None,
            shell=None, expect_fail=False):
        """Allows for calling arbitrary commands.

        Helper for developing purposes, i.e. to quickly implement git-annex
        commands for proof of concept.

        Parameters
        ----------
        files: list of files
        cmd_str: str
            arbitrary command str. `files` is appended to that string.

        Returns
        -------
        stdout, stderr
        """
        cmd = shlex.split(cmd_str + " " + " ".join(files), posix=not on_windows) \
            if isinstance(cmd_str, string_types) \
            else cmd_str + files

        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        return self.cmd_call_wrapper.run(
            cmd,
            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,
            expect_stderr=expect_stderr,
            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)

    @normalize_paths
    def migrate_backend(self, files, backend=None):
        """Changes the backend used for `file`.

        The backend used for the key-value of `files`. Only files currently
        present are migrated.
        Note: There will be no notification if migrating fails due to the
        absence of a file's content!

        Parameters
        ----------
        files: list
            files to migrate.
        backend: str
            specify the backend to migrate to. If none is given, the
            default backend of this instance will be used.
        """

        if self.is_direct_mode():
            raise CommandNotAvailableError(
                'git-annex migrate',
                "Command 'migrate' is not available in direct mode.")
        self._run_annex_command('migrate',
                                annex_options=files,
                                backend=backend)

    @normalize_paths
    def get_file_backend(self, files):
        """Get the backend currently used for file(s).

        Parameters
        ----------
        files: list of str

        Returns
        -------
        list of str
            For each file in input list indicates the used backend by a str
            like "SHA256E" or "MD5".
        """

        return [self.get_file_key(f).split('-')[0] for f in files]

    @property
    def default_backends(self):
        self.config.reload()
        backends = self.config.get("annex.backends", default=None)
        if backends:
            return backends.split()
        else:
            return None

    def fsck(self):
        self._run_annex_command('fsck')

    # TODO: we probably need to override get_file_content, since it returns the
    # symlink's target instead of the actual content.

    # We need --auto and --fast having exposed  TODO
    @normalize_paths(match_return_type=False)  # get a list even in case of a single item
    def copy_to(self, files, remote, options=None, jobs=None):
        """Copy the actual content of `files` to `remote`

        Parameters
        ----------
        files: str or list of str
            path(s) to copy
        remote: str
            name of remote to copy `files` to

        Returns
        -------
        list of str
           files successfully copied
        """

        # find --in here --not --in remote
        # TODO: full support of annex copy options would lead to `files` being
        # optional. This means to check for whether files or certain options are
        # given and fail or just pass everything as is and try to figure out,
        # what was going on when catching CommandError

        if remote not in self.get_remotes():
            raise ValueError("Unknown remote '{0}'.".format(remote))

        options = options[:] if options else []

        # Note:
        # In case of single path, 'annex copy' will fail, if it cannot copy it.
        # With multiple files, annex will just skip the ones, it cannot deal
        # with. We'll do the same and report back what was successful
        # (see return value).
        # Therefore raise telling exceptions before even calling annex:
        if len(files) == 1:
            if not isdir(files[0]):
                self.get_file_key(files[0])

        # TODO: RF -- logic is duplicated with get() -- the only difference
        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?
        if '--key' not in options:
            expected_copys, copy_files = self._get_expected_files(
                files, ['--in', 'here', '--not', '--in', remote])
        else:
            copy_files = files
            assert(len(files) == 1)
            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}

        if not copy_files:
            lgr.debug("No files found needing copying.")
            return []

        if len(copy_files) != len(files):
            lgr.debug("Actually copying %d files", len(copy_files))

        annex_options = ['--to=%s' % remote]
        if options:
            annex_options.extend(shlex.split(options))

        # TODO: provide more meaningful message (possibly aggregating 'note'
        #  from annex failed ones
        results = self._run_annex_command_json(
            'copy',
            opts=annex_options,
            files=files,  # copy_files,
            jobs=jobs,
            expected_entries=expected_copys,
            progress=True
            #log_stdout=True, log_stderr=not log_online,
            #log_online=log_online, expect_stderr=True
        )
        results_list = list(results)
        # XXX this is the only logic different ATM from get
        # check if any transfer failed since then we should just raise an Exception
        # for now to guarantee consistent behavior with non--json output
        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456
        from operator import itemgetter
        failed_copies = [e['file'] for e in results_list if not e['success']]
        good_copies = [
            e['file'] for e in results_list
            if e['success'] and
               e.get('note', '').startswith('to ')  # transfer did happen
        ]
        if failed_copies:
            # TODO: RF for new fancy scheme of outputs reporting
            raise IncompleteResultsError(
                results=good_copies, failed=failed_copies,
                msg="Failed to copy %d file(s)" % len(failed_copies))
        return good_copies

    @property
    def uuid(self):
        """Annex UUID

        Returns
        -------
        str
          Returns a the annex UUID, if there is any, or `None` otherwise.
        """
        if not self._uuid:
            if not self.repo:
                return None
            self._uuid = self.config.get('annex.uuid', default=None)
        return self._uuid

    def get_description(self, uuid=None):
        """Get annex repository description

        Parameters
        ----------
        uuid : str, optional
          For which remote (based on uuid) to report description for

        Returns
        -------
        str or None
          None returned if not found
        """
        info = self.repo_info(fast=True)
        match = \
            (lambda x: x['here']) \
            if uuid is None \
            else (lambda x: x['uuid'] == uuid)

        matches = list(set(chain.from_iterable(
            [
                [r['description'] for r in remotes if match(r)]
                for k, remotes in iteritems(info)
                if k.endswith(' repositories')
            ]
        )))

        if len(matches) == 1:
            # single hit as it should
            return matches[0]
        elif len(matches) == 2:
            lgr.warning(
                "Found multiple hits while searching. Returning first among: %s",
                str(matches)
            )
            return matches[0]
        else:
            return None

    def set_remote_url(self, name, url, push=False):
        """Set the URL a remote is pointing to

        Sets the URL of the remote `name`. Requires the remote to already exist.

        Parameters
        ----------
        name: str
          name of the remote
        url: str
        push: bool
          if True, set the push URL, otherwise the fetch URL;
          if True, additionally set annexurl to `url`, to make sure annex uses
          it to talk to the remote, since access via fetch URL might be
          restricted.
        """

        if push:
            # if we are to set a push url, also set 'annexUrl' for this remote,
            # in order to make git-annex use it, when talking to the remote.
            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)
            var = 'remote.{0}.{1}'.format(name, 'annexurl')
            self.config.set(var, url, where='local', reload=True)
        super(AnnexRepo, self).set_remote_url(name, url, push)

    def get_metadata(self, files, timestamps=False):
        """Query git-annex file metadata

        Parameters
        ----------
        files : str or list(str)
          One or more paths for which metadata is to be queried.
        timestamps: bool, optional
          If True, the output contains a '<metadatakey>-lastchanged'
          key for every metadata item, reflecting the modification
          time, as well as a 'lastchanged' key with the most recent
          modification time of any metadata item.

        Returns
        -------
        generator
          One tuple per file (could be more items than input arguments
          when directories are given). First tuple item is the filename,
          second item is a dictionary with metadata key/value pairs. Note that annex
          metadata tags are stored under the key 'tag', which is a
          regular metadata item that can be manipulated like any other.
        """
        if not files:
            return
        files = assure_list(files)
        opts = ['--json']
        for res in self._run_annex_command_json(
                'metadata', opts=opts, files=files):
            yield (
                res['file'],
                res['fields'] if timestamps else \
                {k: v for k, v in res['fields'].items()
                 if not k.endswith('lastchanged')})

    def set_metadata(
            self, files, reset=None, add=None, init=None,
            remove=None, purge=None, recursive=False):
        """Manipulate git-annex file-metadata

        Parameters
        ----------
        files : str or list(str)
          One or more paths for which metadata is to be manipulated.
          The changes applied to each file item are uniform. However,
          the result may not be uniform across files, depending on the
          actual operation.
        reset : dict, optional
          Metadata items matching keys in the given dict are (re)set
          to the respective values.
        add : dict, optional
          The values of matching keys in the given dict appended to
          any possibly existing values. The metadata keys need not
          necessarily exist before.
        init : dict, optional
          Metadata items for the keys in the given dict are set
          to the respective values, if the key is not yet present
          in a file's metadata.
        remove : dict, optional
          Values in the given dict are removed from the metadata items
          matching the respective key, if they exist in a file's metadata.
          Non-existing values, or keys do not lead to failure.
        purge : list, optional
          Any metadata item with a key matching an entry in the given
          list is removed from the metadata.
        recursive : bool, optional
          If False, fail (with CommandError) when directory paths
          are given as `files`.

        Returns
        -------
        generator
          JSON obj per modified file
        """

        def _genspec(expr, d):
            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]

        args = []
        spec = []
        for expr, d in (('{}={}', reset),
                        ('{}+={}', add),
                        ('{}?={}', init),
                        ('{}-={}', remove)):
            if d:
                spec.extend(_genspec(expr, d))
        # prefix all with '-s' and extend arg list
        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)
        if purge:
            # and all '-r' args
            args.extend(j for i in zip(['-r'] * len(purge), purge)
                        for j in i)
        if not args:
            return

        if recursive:
            args.append('--force')

        # Make sure that batch add/addurl operations are closed so that we can
        # operate on files that were just added.
        self.precommit()

        for jsn in self._run_annex_command_json(
                'metadata',
                args,
                files=files):
            yield jsn

    @staticmethod
    def _is_annex_work_tree_message(out):
        return re.match(
            r'.*This operation must be run in a work tree.*'
            r'git status.*failed in submodule',
            out,
            re.MULTILINE | re.DOTALL | re.IGNORECASE)

# TODO: Why was this commented out?
# @auto_repr
class BatchedAnnexes(dict):
    """Class to contain the registry of active batch'ed instances of annex for
    a repository
    """
    def __init__(self, batch_size=0, git_options=None):
        self.batch_size = batch_size
        self.git_options = git_options or []
        super(BatchedAnnexes, self).__init__()

    def get(self, codename, annex_cmd=None, **kwargs):
        if annex_cmd is None:
            annex_cmd = codename

        git_options = self.git_options + kwargs.pop('git_options', [])
        if self.batch_size:
            git_options += ['-c', 'annex.queuesize=%d' % self.batch_size]

        # START RF/BF: extend codename to respect different options the process
        # is running with
        # TODO: Eventually there should be more RF'ing, since the actually used
        # codenames are partially reflecting this already. Any options used
        # therein should go away, since they are now automatically included.
        options = kwargs.copy()
        options['git_options'] = git_options
        options['annex_cmd'] = annex_cmd
        for key in options:
            codename += ':{0}:{1}'.format(key, options[key])
        # END RF/BF

        if codename not in self:
            # Create a new git-annex process we will keep around
            self[codename] = BatchedAnnex(annex_cmd,
                                          git_options=git_options,
                                          **kwargs)
        return self[codename]

    def clear(self):
        """Override just to make sure we don't rely on __del__ to close all
        the pipes"""
        self.close()
        super(BatchedAnnexes, self).clear()

    def close(self):
        """Close communication to all the batched annexes

        It does not remove them from the dictionary though
        """
        for p in self.values():
            p.close()

    def __del__(self):
        self.close()


def readline_rstripped(stdout):
    #return iter(stdout.readline, b'').next().rstrip()
    return stdout.readline().rstrip()


def readlines_until_ok_or_failed(stdout, maxlines=100):
    """Read stdout until line ends with ok or failed"""
    out = ''
    i = 0
    lgr.log(3, "Trying to receive from %s" % stdout)
    while not stdout.closed:
        i += 1
        if maxlines > 0 and i > maxlines:
            raise IOError("Expected no more than %d lines. So far received: %r" % (maxlines, out))
        lgr.log(2, "Expecting a line")
        line = stdout.readline()
        lgr.log(2, "Received line %r" % line)
        out += line
        if re.match(r'^.*\b(failed|ok)$', line.rstrip()):
            break
    return out.rstrip()


def readline_json(stdout):
    toload = stdout.readline().strip()
    return json_loads(toload) if toload else {}


@auto_repr
class BatchedAnnex(object):
    """Container for an annex process which would allow for persistent communication
    """

    def __init__(self, annex_cmd, git_options=None, annex_options=None, path=None,
                 json=False,
                 output_proc=None):
        if not isinstance(annex_cmd, list):
            annex_cmd = [annex_cmd]
        self.annex_cmd = annex_cmd
        self.git_options = git_options if git_options else []
        annex_options = annex_options if annex_options else []
        self.annex_options = annex_options + (['--json'] if json else [])
        self.path = path
        if output_proc is None:
            output_proc = readline_json if json else readline_rstripped
        self.output_proc = output_proc
        self._process = None
        self._stderr_out = None
        self._stderr_out_fname = None

    def _initialize(self):
        # TODO -- should get all those options about --debug and --backend which are used/composed
        # in AnnexRepo class
        lgr.debug("Initiating a new process for %s" % repr(self))
        cmd = ['git'] + self.git_options + \
              ['annex'] + self.annex_cmd + self.annex_options + ['--batch']  # , '--debug']
        lgr.log(5, "Command: %s" % cmd)
        # TODO: look into _run_annex_command  to support default options such as --debug
        #
        # according to the internet wisdom there is no easy way with subprocess
        # while avoid deadlocks etc.  We would need to start a thread/subprocess
        # to timeout etc
        # kwargs = dict(bufsize=1, universal_newlines=True) if PY3 else {}
        self._stderr_out, self._stderr_out_fname = tempfile.mkstemp()
        self._process = Popen(
            cmd, stdin=PIPE, stdout=PIPE,  stderr=self._stderr_out,
            env=GitRunner.get_git_environ_adjusted(),
            cwd=self.path,
            bufsize=1,
            universal_newlines=True  # **kwargs
        )

    def _check_process(self, restart=False):
        """Check if the process was terminated and restart if restart

        Returns
        -------
        bool
          True if process was alive.
        str
          stderr if any recorded if was terminated
        """
        process = self._process
        ret = True
        ret_stderr = None
        if process and process.poll():
            lgr.warning("Process %s was terminated with returncode %s" % (process, process.returncode))
            ret_stderr = self.close(return_stderr=True)
            ret = False
        if self._process is None and restart:
            lgr.warning("Restarting the process due to previous failure")
            self._initialize()
        return ret, ret_stderr

    def __call__(self, cmds):
        """

        Parameters
        ----------
        cmds : str or tuple or list of (str or tuple)

        Returns
        -------
        str or list
          Output received from annex.  list in case if cmds was a list
        """
        # TODO: add checks -- may be process died off and needs to be reinitiated
        if not self._process:
            self._initialize()

        input_multiple = isinstance(cmds, list)
        if not input_multiple:
            cmds = [cmds]

        output = []

        for entry in cmds:
            if not isinstance(entry, string_types):
                entry = ' '.join(entry)
            entry = entry + '\n'
            lgr.log(5, "Sending %r to batched annex %s" % (entry, self))
            # apparently communicate is just a one time show
            # stdout, stderr = self._process.communicate(entry)
            # according to the internet wisdom there is no easy way with subprocess
            self._check_process(restart=True)
            process = self._process  # _check_process might have restarted it
            process.stdin.write(assure_bytes(entry) if PY2 else entry)
            process.stdin.flush()
            lgr.log(5, "Done sending.")
            still_alive, stderr = self._check_process(restart=False)
            # TODO: we might want to handle still_alive, e.g. to allow for
            #       a number of restarts/resends, but it should be per command
            #       since for some we cannot just resend the same query. But if
            #       it is just a "get"er - we could resend it few times
            # We are expecting a single line output
            # TODO: timeouts etc
            stdout = assure_unicode(self.output_proc(process.stdout)) \
                if not process.stdout.closed else None
            if stderr:
                lgr.warning("Received output in stderr: %r", stderr)
            lgr.log(5, "Received output: %r" % stdout)
            output.append(stdout)

        return output if input_multiple else output[0]

    def __del__(self):
        self.close()

    def close(self, return_stderr=False):
        """Close communication and wait for process to terminate

        Returns
        -------
        str
          stderr output if return_stderr and stderr file was there.
          None otherwise
        """
        ret = None
        if self._stderr_out:
            # close possibly still open fd
            os.fdopen(self._stderr_out).close()
            self._stderr_out = None
        if self._process:
            process = self._process
            lgr.debug(
                "Closing stdin of %s and waiting process to finish", process)
            process.stdin.close()
            process.stdout.close()
            process.wait()
            self._process = None
            lgr.debug("Process %s has finished", process)
        if self._stderr_out_fname and os.path.exists(self._stderr_out_fname):
            if return_stderr:
                with open(self._stderr_out_fname, 'r') as f:
                    ret = f.read()
            # remove the file where we kept dumping stderr
            unlink(self._stderr_out_fname)
            self._stderr_out_fname = None
        return ret


def _get_size_from_perc_complete(count, perc):
    """A helper to get full size if know % and corresponding size"""
    portion = (float(perc) / 100.)
    return int(math.ceil(int(count) / portion)) \
        if portion else 0


class ProcessAnnexProgressIndicators(object):
    """'Filter' for annex --json output to react to progress indicators

    Instance of this beast should be passed into log_stdout option
    for git-annex commands runner
    """

    def __init__(self, expected=None):
        """

        Parameters
        ----------
        expected: dict, optional
           key -> size, expected entries (e.g. downloads)
        """
        # looking forward for multiple downloads at the same time
        self.pbars = {}
        self.total_pbar = None
        self.expected = expected
        self.only_one_expected = expected and len(self.expected) == 1
        self._failed = 0
        self._succeeded = 0
        self.start()

    def start(self):
        if self.expected:
            from datalad.ui import ui
            total = sum(filter(bool, self.expected.values()))
            if len(self.expected) > 1:
                self.total_pbar = ui.get_progressbar(
                    label="Total", total=total)
                self.total_pbar.start()

    def __getitem__(self, id_):
        if self.pbars == 1:
            # regardless what it is, it is the one!
            # will happen e.g. in case of add_url_to_file since filename
            # is not even reported in the status and final msg just
            # reports the key which is not known before
            return self.pbars.items()[0]
        return self.pbars[id_]

    def _update_pbar(self, pbar, new_value):
        """Updates pbar while also updating possibly total pbar"""
        old_value = getattr(pbar, '_old_value', 0)
        # due to http://git-annex.branchable.com/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/?updated
        # we will just skip the first update to avoid possible incorrect
        # reporting
        if not getattr(pbar, '_first_skipped', False):
            setattr(pbar, '_first_skipped', True)
            lgr.log(1, "Skipped first update of pbar %s", pbar)
            return
        setattr(pbar, '_old_value', new_value)
        diff = new_value - old_value
        if diff < 0:
            # so above didn't help!
            # use warnings not lgr.warn since we apparently swallow stuff
            # upstairs!  Also it would take care about issuing it only once
            lgr.debug(
                "Got negative diff for progressbar. old_value=%r, new_value=%r",
                old_value, new_value)
            return
        if self.total_pbar:
            self.total_pbar.update(diff, increment=True)
        pbar.update(new_value)

    def _log_info(self, msg):
        """Helper to log a message, so we need to clear up the pbars first"""
        if self.total_pbar:
            self.total_pbar.clear()
        for pbar in self.pbars.values():
            pbar.clear()
        lgr.info(msg)

    def __call__(self, line):
        try:
            j = json.loads(line)
        except:
            # if we fail to parse, just return this precious thing for
            # possibly further processing
            return line
        # Process some messages which remotes etc might push to us
        if list(j) == ['info']:
            # Just INFO was received without anything else -- we log it at INFO
            info = j['info']
            if info.startswith('PROGRESS-JSON: '):
                j_ = json_loads(info[len('PROGRESS-JSON: '):])
                if ('command' in j_ and 'key' in j_) or 'byte-progress' in j_:
                    j = j_
                else:
                    self._log_info(info)
            else:
                self._log_info(info)
                return

        target_size = None
        if 'command' in j and ('key' in j or 'file' in j):
            # might be the finish line message
            action_item = j.get('key') or j.get('file')
            j_action_id = (j['command'], action_item)
            pbar = self.pbars.pop(j_action_id, None)
            if pbar is None and len(self.pbars) == 1 and self.only_one_expected:
                # it is the only one left - take it!
                pbar = self.pbars.popitem()[1]
            if j.get('success') in {True, 'true'}:
                self._succeeded += 1
                if pbar:
                    self._update_pbar(pbar, pbar.total)
                elif self.total_pbar:
                    # we didn't have a pbar for this download, so total should
                    # get it all at once
                    try:
                        target_size = self.expected[action_item]
                    except:
                        target_size = None
                    if not target_size and 'key' in j:
                        target_size = AnnexRepo.get_size_from_key(j['key'])
                    self.total_pbar.update(target_size, increment=True)
            else:
                lgr.log(5, "Message with failed status: %s" % str(j))
                self._failed += 1

            if self.total_pbar:
                failed_str = (
                    ", " + ansi_colors.color_word("%d failed" % self._failed,
                                                  ansi_colors.RED)) \
                    if self._failed else ''

                self.total_pbar.set_desc(
                    "Total (%d ok%s out of %d)" % (
                        self._succeeded,
                        failed_str,
                        len(self.expected)
                        if self.expected
                        else self._succeeded + self._failed))
                # seems to be of no effect to force it repaint
                self.total_pbar.refresh()

            if pbar:
                pbar.finish()

        if 'byte-progress' not in j:
            # some other thing than progress
            return line

        # so we have a progress indicator, let's deal with it
        action = j['action']
        action_item = action.get('key') or action.get('file')
        action_id = (action['command'], action_item)
        if action_id not in self.pbars:
            # New download!
            from datalad.ui import ui
            from datalad.ui import utils as ui_utils
            # TODO: whenever target size gets reported -- used it!
            # http://git-annex.branchable.com/todo/interface_to_the___34__progress__34___of_annex_operations/#comment-6bbc26aae9867603863050ddcb09a9a0
            # for now deduce from key or approx from '%'
            # TODO: unittest etc to check when we have a relaxed
            # URL without any size known in advance
            if not target_size:
                target_size = \
                    AnnexRepo.get_size_from_key(action.get('key')) or \
                    _get_size_from_perc_complete(
                        j['byte-progress'],
                        j.get('percent-progress', '').rstrip('%')
                    ) or \
                    0
            w = ui_utils.get_console_width()

            if not action_item and self.only_one_expected:
                # must be the one!
                action_item = list(self.expected)[0]

            title = str(action.get('file') or action_item)

            pbar_right = 50
            title_len = w - pbar_right - 4  # (4 for reserve)
            if len(title) > title_len:
                half = title_len//2 - 2
                title = '%s .. %s' % (title[:half], title[-half:])
            pbar = self.pbars[action_id] = ui.get_progressbar(
                label=title, total=target_size)
            pbar.start()

        lgr.log(1, "Updating pbar for action_id=%s. annex: %s.\n",
                action_id, j)
        self._update_pbar(
            self[action_id],
            int(j.get('byte-progress'))
        )

    def finish(self):
        if self.total_pbar:
            self.total_pbar.finish()
            self.total_pbar = None
        if self.pbars:
            lgr.warning("Still have %d active progress bars when stopping",
                        len(self.pbars))
        for pbar in self.pbars.values():
            pbar.finish()
        self.pbars = {}
        self._failed = 0
        self._succeeded = 0

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** `_get_size_from_perc_complete` crashes when percentage is missing

**Description:**
When the percentage value is not provided, the `_get_size_from_perc_complete` function fails instead of handling the missing input gracefully.

**Example Code:**
```python
result = _get_size_from_perc_complete(1, '')
```

**Expected Behavior:**
The function should return `0` when the percentage is not provided.

**Actual Behavior:**
A `ValueError` is raised because it attempts to convert an empty string to a float.
[/ISSUE]
{
    "test_ProcessAnnexProgressIndicators": "PASSED",
    "test_get_size_from_perc_complete": "PASSED",
    "test_AnnexRepo_instance_from_clone": "ERROR",
    "test_AnnexRepo_crippled_filesystem": "ERROR",
    "test_AnnexRepo_set_direct_mode": "ERROR",
    "test_AnnexRepo_annex_proxy": "ERROR",
    "test_AnnexRepo_get_file_key": "ERROR",
    "test_AnnexRepo_get_outofspace": "ERROR",
    "test_AnnexRepo_web_remote": "ERROR",
    "test_AnnexRepo_migrating_backends": "ERROR",
    "test_AnnexRepo_backend_option": "ERROR",
    "test_AnnexRepo_get_file_backend": "ERROR",
    "test_AnnexRepo_on_uninited_annex": "ERROR",
    "test_AnnexRepo_get": "ERROR",
    "test_AnnexRepo_addurl_to_file_batched": "ERROR",
    "test_annexrepo_fake_dates_disables_batched": "ERROR",
    "test_annex_ssh": "ERROR",
    "test_repo_version": "ERROR",
    "test_annex_copy_to": "ERROR",
    "test_annex_drop": "ERROR",
    "test_get_description": "ERROR",
    "test_AnnexRepo_flyweight": "ERROR",
    "test_AnnexRepo_get_toppath": "ERROR",
    "test_AnnexRepo_add_submodule": "ERROR",
    "test_AnnexRepo_status": "ERROR",
    "test_AnnexRepo_addurl_batched_and_set_metadata": "ERROR",
    "test_AnnexRepo_flyweight_monitoring_inode": "ERROR",
    "test_AnnexRepo_instance_from_existing": "FAILED",
    "test_AnnexRepo_instance_brand_new": "FAILED",
    "test_AnnexRepo_is_direct_mode": "FAILED",
    "test_AnnexRepo_is_direct_mode_gitrepo": "FAILED",
    "test_AnnexRepo_get_remote_na": "FAILED",
    "test_find_batch_equivalence": "FAILED",
    "test_repo_info": "FAILED",
    "test_AnnexRepo_always_commit": "FAILED",
    "test_AnnexRepo_commit": "FAILED",
    "test_AnnexRepo_add_to_annex": "FAILED",
    "test_AnnexRepo_add_to_git": "FAILED",
    "test_AnnexRepo_add_unexpected_direct_mode": "FAILED",
    "test_annex_backends": "FAILED",
    "test_annex_remove": "FAILED",
    "test_annex_get_annexed_files": "FAILED",
    "test_annex_add_no_dotfiles": "FAILED",
    "test_annex_version_handling": "FAILED",
    "test_AnnexRepo_dirty": "FAILED",
    "test_AnnexRepo_set_remote_url": "FAILED",
    "test_wanted": "FAILED",
    "test_AnnexRepo_metadata": "FAILED",
    "test_change_description": "FAILED",
    "test_AnnexRepo_get_corresponding_branch": "FAILED",
    "test_AnnexRepo_get_tracking_branch": "FAILED",
    "test_AnnexRepo_is_managed_branch": "FAILED",
    "test_fake_is_not_special": "FAILED",
    "test_fake_dates": "FAILED"
}
diff --git a/datalad/support/annexrepo.py b/datalad/support/annexrepo.py
index a914b8194..d603e9524 100644
--- a/datalad/support/annexrepo.py
+++ b/datalad/support/annexrepo.py
@@ -2112,8 +2112,8 @@ class AnnexRepo(GitRepo, RepoInterface):
                     % str(out_json))
             # Make the output's structure match bcmd's.
             out_json = out_json[0]
-            # Don't capture stderr, since download progress provided by wget uses
-            # stderr.
+            # Don't capture stderr, since download progress provided by wget
+            # uses stderr.
         else:
             options += ['--with-files']
             if backend:
@@ -3661,7 +3661,11 @@ class BatchedAnnex(object):
 
 def _get_size_from_perc_complete(count, perc):
     """A helper to get full size if know % and corresponding size"""
-    portion = (float(perc) / 100.)
+
+    try:
+        portion = (float(perc) / 100.)
+    except ValueError:
+        portion = None
     return int(math.ceil(int(count) / portion)) \
         if portion else 0
 

{"datalad/support/annexrepo.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Interface to git-annex by Joey Hess.\n\nFor further information on git-annex see https://git-annex.branchable.com/.\n\n\"\"\"\n\nimport json\nimport logging\nimport math\nimport os\nimport re\nimport shlex\nimport tempfile\nimport time\n\nfrom itertools import chain\nfrom os import linesep\nfrom os.path import curdir\nfrom os.path import join as opj\nfrom os.path import exists\nfrom os.path import islink\nfrom os.path import realpath\nfrom os.path import lexists\nfrom os.path import isdir\nfrom os.path import isabs\nfrom os.path import relpath\nfrom os.path import normpath\nfrom subprocess import Popen, PIPE\nfrom multiprocessing import cpu_count\nfrom weakref import WeakValueDictionary\n\nfrom six import string_types, PY2\nfrom six import iteritems\nfrom six.moves import filter\nfrom git import InvalidGitRepositoryError\n\nfrom datalad import ssh_manager\nfrom datalad.dochelpers import exc_str\nfrom datalad.dochelpers import borrowdoc\nfrom datalad.dochelpers import borrowkwargs\nfrom datalad.utils import linux_distribution_name\nfrom datalad.utils import nothing_cm\nfrom datalad.utils import auto_repr\nfrom datalad.utils import on_windows\nfrom datalad.utils import swallow_logs\nfrom datalad.utils import assure_list\nfrom datalad.utils import _path_\nfrom datalad.utils import generate_chunks\nfrom datalad.utils import CMD_MAX_ARG\nfrom datalad.utils import assure_unicode, assure_bytes\nfrom datalad.utils import make_tempfile\nfrom datalad.utils import unlink\nfrom datalad.support.json_py import loads as json_loads\nfrom datalad.cmd import GitRunner\n\n# imports from same module:\nfrom .repo import RepoInterface\nfrom .gitrepo import GitRepo\nfrom .gitrepo import NoSuchPathError\nfrom .gitrepo import _normalize_path\nfrom .gitrepo import normalize_path\nfrom .gitrepo import normalize_paths\nfrom .gitrepo import GitCommandError\nfrom .gitrepo import to_options\nfrom . import ansi_colors\nfrom .external_versions import external_versions\nfrom .exceptions import CommandNotAvailableError\nfrom .exceptions import CommandError\nfrom .exceptions import FileNotInAnnexError\nfrom .exceptions import FileInGitError\nfrom .exceptions import FileNotInRepositoryError\nfrom .exceptions import AnnexBatchCommandError\nfrom .exceptions import InsufficientArgumentsError\nfrom .exceptions import OutOfSpaceError\nfrom .exceptions import RemoteNotAvailableError\nfrom .exceptions import OutdatedExternalDependency\nfrom .exceptions import MissingExternalDependency\nfrom .exceptions import IncompleteResultsError\nfrom .exceptions import AccessDeniedError\nfrom .exceptions import AccessFailedError\n\nlgr = logging.getLogger('datalad.annex')\n\n# Limit to # of CPUs and up to 8, but at least 3 to start with\nN_AUTO_JOBS = min(8, max(3, cpu_count()))\n\n\nclass AnnexRepo(GitRepo, RepoInterface):\n    \"\"\"Representation of an git-annex repository.\n\n    Paths given to any of the class methods will be interpreted as relative\n    to PWD, in case this is currently beneath AnnexRepo's base dir\n    (`self.path`). If PWD is outside of the repository, relative paths\n    will be interpreted as relative to `self.path`. Absolute paths will be\n    accepted either way.\n    \"\"\"\n\n    # Begin Flyweight:\n    _unique_instances = WeakValueDictionary()\n    # End Flyweight:\n\n    # Web remote has a hard-coded UUID we might (ab)use\n    WEB_UUID = \"00000000-0000-0000-0000-000000000001\"\n\n    # To be assigned and checked to be good enough upon first call to AnnexRepo\n    # 6.20160923 -- --json-progress for get\n    # 6.20161210 -- annex add  to add also changes (not only new files) to git\n    # 6.20170220 -- annex status provides --ignore-submodules\n    # 6.20180416 -- annex handles unicode filenames more uniformly\n    # 6.20180913 -- annex fixes all known to us issues for v6\n    GIT_ANNEX_MIN_VERSION = '6.20180913'\n    git_annex_version = None\n\n    # Class wide setting to allow insecure URLs. Used during testing, since\n    # git annex 6.20180626 those will by default be not allowed for security\n    # reasons\n    _ALLOW_LOCAL_URLS = False\n\n    def __init__(self, path, url=None, runner=None,\n                 direct=None, backend=None, always_commit=True, create=True,\n                 init=False, batch_size=None, version=None, description=None,\n                 git_opts=None, annex_opts=None, annex_init_opts=None,\n                 repo=None, fake_dates=False):\n        \"\"\"Creates representation of git-annex repository at `path`.\n\n        AnnexRepo is initialized by giving a path to the annex.\n        If no annex exists at that location, a new one is created.\n        Optionally give url to clone from.\n\n        Parameters\n        ----------\n        path: str\n          Path to git-annex repository. In case it's not an absolute path, it's\n          relative to PWD\n        url: str, optional\n          url to the to-be-cloned repository. Requires valid git url\n          according to\n          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .\n        runner: Runner, optional\n          Provide a Runner in case AnnexRepo shall not create it's own.\n          This is especially needed in case of desired dry runs.\n        direct: bool, optional\n          If True, force git-annex to use direct mode\n        backend: str, optional\n          Set default backend used by this annex. This does NOT affect files,\n          that are already annexed nor will it automatically migrate files,\n          hat are 'getted' afterwards.\n        create: bool, optional\n          Create and initialize an annex repository at path, in case\n          there is none. If set to False, and this repository is not an annex\n          repository (initialized or not), an exception is raised.\n        init: bool, optional\n          Initialize git-annex repository (run \"git annex init\") if path is an\n          annex repository which just was not yet initialized by annex (e.g. a\n          fresh git clone). Note that if `create=True`, then initialization\n          would happen\n        batch_size: int, optional\n          If specified and >0, instructs annex to batch this many commands before\n          annex adds acts on git repository (e.g. adds them them to index for addurl).\n        version: int, optional\n          If given, pass as --version to `git annex init`\n        description: str, optional\n          Short description that humans can use to identify the\n          repository/location, e.g. \"Precious data on my laptop\"\n        \"\"\"\n        if self.git_annex_version is None:\n            self._check_git_annex_version()\n\n        # initialize\n        self._uuid = None\n        self._annex_common_options = []\n        # Workaround for per-call config issue with git 2.11.0\n        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False\n        self.GIT_DIRECT_MODE_PROXY = False\n\n        if annex_opts or annex_init_opts:\n            lgr.warning(\"TODO: options passed to git-annex and/or \"\n                        \"git-annex-init are currently ignored.\\n\"\n                        \"options received:\\n\"\n                        \"git-annex: %s\\ngit-annex-init: %s\" %\n                        (annex_opts, annex_init_opts))\n\n        fix_it = False\n        try:\n            super(AnnexRepo, self).__init__(path, url, runner=runner,\n                                            create=create, repo=repo,\n                                            git_opts=git_opts,\n                                            fake_dates=fake_dates)\n        except GitCommandError as e:\n            if create and \"Clone succeeded, but checkout failed.\" in str(e):\n                lgr.warning(\"Experienced issues while cloning. \"\n                            \"Trying to fix it, using git-annex-fsck.\")\n                fix_it = True\n            else:\n                raise e\n\n        # Below while setting for direct mode workaround, we change\n        # _GIT_COMMON_OPTIONS.\n        # But we should not pass workarounds such as --worktree=.\n        # -c core.bare=False to git annex commands, so for their\n        # invocation we will keep and use pristine version of the\n        # common options\n        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]\n\n        # check for possible SSH URLs of the remotes in order to set up\n        # shared connections:\n        for r in self.get_remotes():\n            for url in [self.get_remote_url(r),\n                        self.get_remote_url(r, push=True)]:\n                if url is not None:\n                    self._set_shared_connection(r, url)\n\n        self.always_commit = always_commit\n\n        if version is None:\n            version = self.config.get(\"datalad.repo.version\", None)\n            # we might get an empty string here\n            # TODO: if we use obtain() instead, we get an error complaining\n            # '' cannot be converted to int (via Constraint as defined for\n            # \"datalad.repo.version\" in common_cfg\n            # => Allow conversion to result in None?\n            if not version:\n                version = None\n\n        if fix_it:\n            self._init(version=version, description=description)\n            self.fsck()\n\n        # Check whether an annex already exists at destination\n        # XXX this doesn't work for a submodule!\n        if not AnnexRepo.is_valid_repo(self.path):\n            # so either it is not annex at all or just was not yet initialized\n            if self.is_with_annex():\n                # it is an annex repository which was not initialized yet\n                if create or init:\n                    lgr.debug('Annex repository was not yet initialized at %s.'\n                              ' Initializing ...' % self.path)\n                    self._init(version=version, description=description)\n            elif create:\n                lgr.debug('Initializing annex repository at %s...' % self.path)\n                self._init(version=version, description=description)\n            else:\n                raise RuntimeError(\"No annex found at %s.\" % self.path)\n\n        self._direct_mode = None  # we don't know yet\n\n        # If we are in direct mode already, we need to make\n        # this instance aware of that. This especially means, that we need to\n        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().\n        # Could happen in case we didn't specify anything, but annex forced\n        # direct mode due to FS or an already existing repo was in direct mode,\n        if self._is_direct_mode_from_config():\n            self.set_direct_mode()\n\n        # - only force direct mode; don't force indirect mode\n        # - parameter `direct` has priority over config\n        if direct is None:\n            direct = (create or init) and \\\n                     self.config.getbool(\"datalad\", \"repo.direct\", default=False)\n        self._direct_mode = None  # we don't know yet\n        if direct and not self.is_direct_mode():\n            # direct mode is available below version 6 repos only.\n            # Note: If 'annex.version' is missing in .git/config for some\n            # reason, we need to try to set direct mode:\n            repo_version = self.config.getint(\"annex\", \"version\")\n            if (repo_version is None) or (repo_version < 6):\n                lgr.debug(\"Switching to direct mode (%s).\" % self)\n                self.set_direct_mode()\n            else:\n                # TODO: This may change to either not being a warning and/or\n                # to use 'git annex unlock' instead.\n                lgr.warning(\"direct mode not available for %s. Ignored.\" % self)\n\n        self._batched = BatchedAnnexes(\n            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)\n\n        # set default backend for future annex commands:\n        # TODO: Should the backend option of __init__() also migrate\n        # the annex, in case there are annexed files already?\n        if backend:\n            self.set_default_backend(backend, persistent=True)\n\n        if self._ALLOW_LOCAL_URLS:\n            self._allow_local_urls()\n\n    def _allow_local_urls(self):\n        \"\"\"Allow URL schemes and addresses which potentially could be harmful.\n\n        For now it is internal method used within tests only\n        \"\"\"\n        # from annex 6.20180626 file:/// and http://localhost access isn't\n        # allowed by default\n        self.config.add(\n            'annex.security.allowed-url-schemes', 'http https file',\n            'local')\n        self.config.add(\n            'annex.security.allowed-http-addresses', 'all',\n            'local')\n\n    def set_default_backend(self, backend, persistent=True, commit=True):\n        \"\"\"Set default backend\n\n        Parameters\n        ----------\n        backend : str\n        persistent : bool, optional\n          If persistent, would add/commit to .gitattributes. If not -- would\n          set within .git/config\n        \"\"\"\n        # TODO: 'annex.backends' actually is a space separated list.\n        # Figure out, whether we want to allow for a list here or what to\n        # do, if there is sth in that setting already\n        if persistent:\n            # could be set in .gitattributes or $GIT_DIR/info/attributes\n            if 'annex.backend' in self.get_gitattributes('.')['.']:\n                lgr.debug(\n                    \"Not (re)setting backend since seems already set in git attributes\"\n                )\n            else:\n                lgr.debug(\"Setting annex backend to %s (persistently)\", backend)\n                self.config.set('annex.backends', backend, where='local')\n                git_attributes_file = _path_(self.path, '.gitattributes')\n                git_attributes = ''\n                if exists(git_attributes_file):\n                    with open(git_attributes_file) as f:\n                        git_attributes = f.read()\n                with open(git_attributes_file, 'a') as f:\n                    if git_attributes and not git_attributes.endswith(os.linesep):\n                        f.write(os.linesep)\n                    f.write('* annex.backend=%s%s' % (backend, os.linesep))\n                self.add(git_attributes_file, git=True)\n                if commit:\n                    self.commit(\n                        \"Set default backend for all files to be %s\" % backend,\n                        _datalad_msg=True,\n                        files=[git_attributes_file]\n                    )\n        else:\n            lgr.debug(\"Setting annex backend to %s (in .git/config)\", backend)\n            self.config.set('annex.backends', backend, where='local')\n\n    def __del__(self):\n        try:\n            if hasattr(self, '_batched') and self._batched is not None:\n                self._batched.close()\n        except TypeError as e:\n            # Workaround:\n            # most likely something wasn't accessible anymore; doesn't really\n            # matter since we wanted to delete it anyway.\n            #\n            # Nevertheless, in some cases might be an issue and it is a strange\n            # thing to happen, since we check for things being None herein as\n            # well as in super class __del__;\n            # At least log it:\n            lgr.debug(exc_str(e))\n        try:\n            super(AnnexRepo, self).__del__()\n        except TypeError as e:\n            # see above\n            lgr.debug(exc_str(e))\n\n    def _set_shared_connection(self, remote_name, url):\n        \"\"\"Make sure a remote with SSH URL uses shared connections.\n\n        Set ssh options for annex on a per call basis, using\n        '-c remote.<name>.annex-ssh-options'.\n\n        Note\n        ----\n        There's currently no solution for using these connections, if the SSH\n        URL is just connected to a file instead of a remote\n        (`annex addurl` for example).\n\n        Parameters\n        ----------\n        remote_name: str\n        url: str\n        \"\"\"\n        from datalad.support.network import is_ssh\n        # Note:\n        #\n        # before any possible invocation of git-annex\n        # Temporary approach to ssh connection sharing:\n        # Register every ssh remote with the corresponding control master.\n        # Issues:\n        # - currently overwrites existing ssh config of the remote\n        # - request SSHConnection instance and write config even if no\n        #   connection needed (but: connection is not actually created/opened)\n        # - no solution for a ssh url of a file (annex addurl)\n\n        if is_ssh(url):\n            c = ssh_manager.get_connection(url)\n            ssh_cfg_var = \"remote.{0}.annex-ssh-options\".format(remote_name)\n            # options to add:\n            # Note: must use -S to overload -S provided by annex itself\n            # if we provide -o ControlPath=... it is not in effect\n            # Note: ctrl_path must not contain spaces, since it seems to be\n            # impossible to anyhow guard them here\n            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/\n            cfg_string = \"-o ControlMaster=auto -S %s\" % c.ctrl_path\n            # read user-defined options from .git/config:\n            cfg_string_old = self.config.get(ssh_cfg_var, None)\n            self._annex_common_options += \\\n                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'\n                       ''.format(remote_name,\n                                 (cfg_string_old + \" \") if cfg_string_old else \"\",\n                                 cfg_string\n                                 )]\n\n    def is_managed_branch(self, branch=None):\n        \"\"\"Whether `branch` is managed by git-annex.\n\n        ATM this returns true in direct mode (branch 'annex/direct/my_branch')\n        and if on an adjusted branch (annex v6 repository:\n        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'\n\n        Note: The term 'managed branch' is used to make clear it's meant to be\n        more general than the v6 'adjusted branch'.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; default: active branch\n\n        Returns\n        -------\n        bool\n          True if on a managed branch, False otherwise\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n        # Note: `branch` might still be None, due to detached HEAD\n        # (or no checkout at all)\n        if branch and \\\n            (branch.startswith('annex/direct/') or\n             branch.startswith('adjusted/')):\n            return True\n        return False\n\n    def get_corresponding_branch(self, branch=None):\n        \"\"\"In case of a managed branch, get the corresponding one.\n\n        If `branch` is not a managed branch, return that branch without any\n        changes.\n\n        Note: Since default for `branch` is the active branch,\n        `get_corresponding_branch()` is equivalent to `get_active_branch()` if\n        the active branch is not a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          name of the branch; defaults to active branch\n\n        Returns\n        -------\n        str\n          name of the corresponding branch if there is any, name of the queried\n          branch otherwise.\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        if self.is_managed_branch(branch):\n            if branch.startswith('annex/direct/'):\n                cor_branch = branch[13:]\n            elif branch.startswith('adjusted/'):\n                if branch.endswith('(unlocked)'):\n                    cor_branch = branch[9:-10]\n                elif branch.endswith('(fixed)'):\n                    cor_branch = branch[9:-7]\n                else:\n                    cor_branch = branch[9:]\n                    lgr.warning(\"Unexpected naming of adjusted branch '{}'.{}\"\n                                \"Assuming '{}' to be the corresponding branch.\"\n                                \"\".format(branch, linesep, cor_branch))\n            else:\n                raise NotImplementedError(\n                    \"Detection of annex-managed branch '{}' follows a pattern \"\n                    \"not implemented herein.\".format(branch))\n            return cor_branch\n\n        else:\n            return branch\n\n    def get_tracking_branch(self, branch=None, corresponding=True):\n        \"\"\"Get the tracking branch for `branch` if there is any.\n\n        By default returns the tracking branch of the corresponding branch if\n        `branch` is a managed branch.\n\n        Parameters\n        ----------\n        branch: str\n          local branch to look up. If none is given, active branch is used.\n        corresponding: bool\n          If True actually look up the corresponding branch of `branch` (also if\n          `branch` isn't explicitly given)\n\n        Returns\n        -------\n        tuple\n            (remote or None, refspec or None) of the tracking branch\n        \"\"\"\n\n        if branch is None:\n            branch = self.get_active_branch()\n\n        return super(AnnexRepo, self).get_tracking_branch(\n                        branch=self.get_corresponding_branch(branch)\n                        if corresponding else branch)\n\n    def _submodules_dirty_direct_mode(self,\n            untracked=True, deleted=True, modified=True, added=True,\n            type_changed=True, path=None):\n        \"\"\"Get modified submodules\n\n        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/\n\n        This is using git-annex-status with --ignore-submodules to not let\n        git-status try to recurse into annex submodules without a working tree.\n        Therefore we need to do the recursion on our own.\n\n        Note, that added submodules will just be reported dirty. It's at very\n        least difficult to distinguish whether a submodule in direct mode was\n        just added or modified. ATM not worth the effort, I think.\n        This is leads to a bit inconsistent reportings by AnnexRepo.status()\n        whenever it needs to call this subroutine and there are added submodules.\n\n        Intended to be used by AnnexRepo.status() internally.\n        \"\"\"\n\n        # Note: We do a lazy recursion. The only thing we need to know is\n        # whether or not a submodule is to be reported dirty. Once we already\n        # know it is, there's no need to go any deeper in the hierarchy.\n        # Apart from better performance, this also allows us to inspect each\n        # submodule separately, and therefore be able to deal with mixed\n        # hierarchies of git and annex submodules!\n\n        modified_subs = []\n        for sm in self.get_submodules():\n            sm_dirty = False\n\n            # First check for changes committed in the submodule, using\n            # git submodule summary -- path,\n            # since this can't be detected from within the submodule.\n            if self.is_submodule_modified(sm.name):\n                sm_dirty = True\n\n            # check state of annex submodules, that might be in direct mode\n            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),\n                                         allow_noninitialized=False):\n\n                sm_repo = AnnexRepo(opj(self.path, sm.path),\n                                    create=False, init=False)\n\n                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,\n                                               modified=modified, added=added,\n                                               type_changed=type_changed,\n                                               submodules=False, path=path)\n                if any([bool(sm_status[i]) for i in sm_status]):\n                    sm_dirty = True\n\n            # check state of submodule, that is a plain git or not an\n            # initialized annex, which we can safely treat as a plain git, too.\n            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):\n                sm_repo = GitRepo(opj(self.path, sm.path))\n\n                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters\n                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,\n                                    working_tree=deleted or modified or added or type_changed,\n                                    untracked_files=untracked,\n                                    submodules=False, path=path):\n                    sm_dirty = True\n            else:\n                # uninitialized submodule\n                # it can't be dirty and we can't recurse any deeper:\n                continue\n\n            if sm_dirty:\n                # the submodule itself is dirty\n                modified_subs.append(sm.path)\n            else:\n                # the submodule itself is clean, recurse:\n                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.\n                # we need get_status and this recursion method to be available\n                # to both classes. Issue: We need to be able to come back to\n                # AnnexRepo from GitRepo if there's again an annex beneath. But\n                # we can't import AnnexRepo in gitrepo.py.\n                modified_subs.extend(\n                    sm_repo._submodules_dirty_direct_mode(\n                        untracked=untracked, deleted=deleted,\n                        modified=modified, added=added,\n                        type_changed=type_changed, path=path\n                    ))\n\n        return modified_subs\n\n    def get_status(self, untracked=True, deleted=True, modified=True, added=True,\n                   type_changed=True, submodules=True, path=None):\n        \"\"\"Return various aspects of the status of the annex repository\n\n        Note: Under certain circumstances newly added submodules might be\n        reported as 'modified' rather tha 'added'.\n        See `AnnexRepo._submodules_dirty_direct_mode` for details.\n\n        Parameters\n        ----------\n        untracked\n        deleted\n        modified\n        added\n        type_changed\n        submodules\n        path\n\n        Returns\n        -------\n\n        \"\"\"\n\n        self.precommit()\n\n        options = assure_list(path) if path else []\n        if not submodules:\n            options.extend(to_options(ignore_submodules='all'))\n\n        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)\n        # internal call to 'git status' by 'git annex status' will fail\n        # in submodules without a working tree (direct mode)\n        # How to catch this case depends on annex version, since annex\n        # exits zero until version 6.20170307\n\n        def _fake_exception_wrapper(self, options_):\n            \"\"\"generate a faked `CommandError` from logged stderr output\"\"\"\n\n            # this is for use with older annex, which didn't exit non-zero\n            # in case of the failure we are interested in\n\n            # TODO: If we are to keep this workaround (we probably rely on a\n            # newer annex anyway), we should not use swallow_logs, since we\n            # actually don't want to swallow it, but inspect it. Use a proper\n            # handler/filter for the logger instead to not create temp files via\n            # swallow_logs\n\n            old_log_state = self.cmd_call_wrapper.log_outputs\n            self.cmd_call_wrapper._log_opts['outputs'] = True\n\n            with swallow_logs(new_level=logging.ERROR) as cml:\n                # Note, that _run_annex_command_json returns a generator\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options_, expect_stderr=False))\n            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state\n            if \"fatal:\" in cml.out:\n                raise CommandError(cmd=\"git annex status\",\n                                   msg=cml.out, stderr=cml.out)\n            return json_list\n\n        try:\n            if self.git_annex_version < '6.20170307':\n                json_list = _fake_exception_wrapper(self, options_=options)\n            else:\n                json_list = \\\n                    list(self._run_annex_command_json(\n                        'status', opts=options, expect_stderr=False))\n        except CommandError as e:\n            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):\n                lgr.debug(\"git-annex-status failed probably due to submodule in\"\n                          \" direct mode. Trying to workaround.\")\n                # try again, ignoring submodules:\n                options = [path] if path else []\n                options.extend(to_options(ignore_submodules='all'))\n                json_list = list(\n                    self._run_annex_command_json('status', opts=options)\n                )\n                # separately get modified submodules:\n                m_subs = \\\n                    self._submodules_dirty_direct_mode(untracked=untracked,\n                                                       deleted=deleted,\n                                                       modified=modified,\n                                                       added=added,\n                                                       type_changed=type_changed,\n                                                       path=path)\n                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)\n            else:\n                # not the known bug we want to catch\n                raise e\n\n        # END workaround\n\n        key_mapping = [(untracked, 'untracked', '?'),\n                       (deleted, 'deleted', 'D'),\n                       (modified, 'modified', 'M'),\n                       (added, 'added', 'A'),\n                       (type_changed, 'type_changed', 'T')]\n        from datalad.utils import with_pathsep\n        return {key: [with_pathsep(i['file'])\n                      if isdir(opj(self.path, i['file'])) else i['file']\n                      # for consistency with 'git status' return directories\n                      # with trailing path separator\n                      for i in json_list if i['status'] == st]\n                for cond, key, st in key_mapping if cond}\n\n    @borrowdoc(GitRepo)\n    def is_dirty(self, index=True, working_tree=False, untracked_files=True,\n                 submodules=True, path=None):\n        # TODO: Add doc on how this differs from GitRepo.is_dirty()\n        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()\n\n        if working_tree:\n            # Note: annex repos don't always have a git working tree and the\n            # behaviour in direct mode or V6 repos is fundamentally different\n            # from that concept. There are no unstaged changes in direct mode\n            # for example. Therefore the need to call this method with\n            # 'working_tree=True' indicates invalid assumptions in the\n            # calling code.\n\n            # TODO: Better exception. InvalidArgumentError or sth ...\n            raise CommandNotAvailableError(\n                \"Querying a git-annex repository for a clean/dirty \"\n                \"working tree is an invalid concept.\")\n        # Again note, that 'annex status' isn't distinguishing staged and\n        # unstaged changes, since this makes little sense for an annex repo\n        # in general. Therefore we use only 'index' and 'untracked_files' to\n        # specify what kind of dirtyness we are interested in:\n        status = self.get_status(untracked=untracked_files, deleted=index,\n                                 modified=index, added=index,\n                                 type_changed=index, submodules=submodules,\n                                 path=path)\n        return any([bool(status[i]) for i in status])\n\n    @property\n    def untracked_files(self):\n        \"\"\"Get a list of untracked files\n        \"\"\"\n        return self.get_status(untracked=True, deleted=False, modified=False,\n                               added=False, type_changed=False, submodules=False,\n                               path=None)['untracked']\n\n    @classmethod\n    def _check_git_annex_version(cls):\n        ver = external_versions['cmd:annex']\n        # in case it is missing\n        if linux_distribution_name in {'debian', 'ubuntu'}:\n            msg = \"Install  git-annex-standalone  from NeuroDebian \" \\\n                  \"(http://neuro.debian.net)\"\n        else:\n            msg = \"Visit http://git-annex.branchable.com/install/\"\n        exc_kwargs = dict(\n            name=\"git-annex\",\n            msg=msg,\n            ver=cls.GIT_ANNEX_MIN_VERSION\n        )\n        if not ver:\n            raise MissingExternalDependency(**exc_kwargs)\n        elif ver < cls.GIT_ANNEX_MIN_VERSION:\n            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)\n        cls.git_annex_version = ver\n\n    @staticmethod\n    def get_size_from_key(key):\n        \"\"\"A little helper to obtain size encoded in a key\"\"\"\n        if not key:\n            return None\n        try:\n            size_str = key.split('-', 2)[1].lstrip('s')\n        except IndexError:\n            # has no 2nd field in the key\n            return None\n        return int(size_str) if size_str.isdigit() else None\n\n    @normalize_path\n    def get_file_size(self, path):\n        fpath = opj(self.path, path)\n        return 0 if not exists(fpath) else os.stat(fpath).st_size\n\n    # TODO: Once the PR containing super class 'Repo' was merged, move there and\n    # melt with GitRepo.get_toppath including tests for both\n    @classmethod\n    def get_toppath(cls, path, follow_up=True, git_options=None):\n        \"\"\"Return top-level of a repository given the path.\n\n        Parameters\n        -----------\n        follow_up : bool\n          If path has symlinks -- they get resolved by git.  If follow_up is\n          True, we will follow original path up until we hit the same resolved\n          path.  If no such path found, resolved one would be returned.\n        git_options: list of str\n          options to be passed to the git rev-parse call\n\n        Return None if no parent directory contains a git repository.\n        \"\"\"\n\n        # first try plain git result:\n        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n                                      git_options=git_options)\n        if toppath == '':\n            # didn't fail, so git itself didn't come to the conclusion\n            # there is no repo, but we have no actual result;\n            # might be an annex in direct mode\n            if git_options is None:\n                git_options = []\n            # TODO: Apparently doesn't work with git 2.11.0\n            # Note: Since we are in a classmethod, GitRepo.get_toppath uses\n            # Runner directly instead of _git_custom_command, which is why the\n            # common mechanics for direct mode are not applied.\n            # This is why there is no solution for git 2.11 yet\n\n            # Note 2: Actually, the above issue is irrelevant. The git\n            # executable has no repository it is bound to, since it's the\n            # purpose of the call to find this repository. Therefore\n            # core.bare=False has no effect at all.\n\n            # Disabled. See notes.\n            # git_options.extend(['-c', 'core.bare=False'])\n            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,\n            #                               git_options=git_options)\n\n            # basically a copy of code in GitRepo.get_toppath\n            # except it uses 'git rev-parse --git-dir' as a workaround for\n            # direct mode:\n\n            from os.path import dirname\n            from os import pardir\n\n            cmd = ['git']\n            if git_options:\n                cmd.extend(git_options)\n\n            cmd.append(\"rev-parse\")\n            if external_versions['cmd:git'] >= '2.13.0':\n                cmd.append(\"--absolute-git-dir\")\n            else:\n                cmd.append(\"--git-dir\")\n\n            try:\n                toppath, err = GitRunner().run(\n                    cmd,\n                    cwd=path,\n                    log_stdout=True, log_stderr=True,\n                    expect_fail=True, expect_stderr=True)\n                toppath = toppath.rstrip('\\n\\r')\n            except CommandError:\n                return None\n            except OSError:\n                toppath = AnnexRepo.get_toppath(dirname(path),\n                                                follow_up=follow_up,\n                                                git_options=git_options)\n\n            if external_versions['cmd:git'] < '2.13.0':\n                # we got a path relative to `path` instead of an absolute one\n                toppath = opj(path, toppath)\n\n            # we got the git-dir. Assuming the root dir we are looking for is\n            # one level up:\n            toppath = realpath(normpath(opj(toppath, pardir)))\n\n            if follow_up:\n                path_ = path\n                path_prev = \"\"\n                while path_ and path_ != path_prev:  # on top /.. = /\n                    if realpath(path_) == toppath:\n                        toppath = path_\n                        break\n                    path_prev = path_\n                    path_ = dirname(path_)\n\n        return toppath\n\n    @classmethod\n    def is_valid_repo(cls, path, allow_noninitialized=False):\n        \"\"\"Return True if given path points to an annex repository\n        \"\"\"\n        # Note: default value for allow_noninitialized=False is important\n        # for invalidating an instance via self._flyweight_invalid. If this is\n        # changed, we also need to override _flyweight_invalid and explicitly\n        # pass allow_noninitialized=False!\n\n        def git_file_has_annex(p):\n            \"\"\"Return True if `p` contains a .git file, that points to a git\n            dir with a subdir 'annex'\"\"\"\n            _git = opj(p, '.git')\n            if not os.path.isfile(_git):\n                return False\n            with open(_git, \"r\") as f:\n                line = f.readline()\n                if line.startswith(\"gitdir: \"):\n                    return exists(opj(p, line[8:], 'annex'))\n                else:\n                    lgr.debug(\"Invalid .git file: %s\", _git)\n                    return False\n\n        initialized_annex = GitRepo.is_valid_repo(path) and \\\n            (exists(opj(path, '.git', 'annex')) or\n             git_file_has_annex(path))\n\n        if allow_noninitialized:\n            try:\n                return initialized_annex \\\n                    or GitRepo(path, create=False, init=False).is_with_annex()\n            except (NoSuchPathError, InvalidGitRepositoryError):\n                return False\n        else:\n            return initialized_annex\n\n    def add_remote(self, name, url, options=None):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n        super(AnnexRepo, self).add_remote(name, url, options if options else [])\n        self._set_shared_connection(name, url)\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Overrides method from GitRepo in order to set\n        remote.<name>.annex-ssh-options in case of a SSH remote.\"\"\"\n\n        super(AnnexRepo, self).set_remote_url(name, url, push=push)\n        self._set_shared_connection(name, url)\n\n    def set_remote_dead(self, name):\n        \"\"\"Announce to annex that remote is \"dead\"\n        \"\"\"\n        return self._annex_custom_command([], [\"git\", \"annex\", \"dead\", name])\n\n    def is_remote_annex_ignored(self, remote):\n        \"\"\"Return True if remote is explicitly ignored\"\"\"\n        return self.config.getbool(\n            'remote.{}'.format(remote), 'annex-ignore',\n            default=False\n        )\n\n    def is_special_annex_remote(self, remote, check_if_known=True):\n        \"\"\"Return either remote is a special annex remote\n\n        Decides based on the presence of diagnostic annex- options\n        for the remote\n        \"\"\"\n        if check_if_known:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(remote)\n        sec = 'remote.{}'.format(remote)\n        for opt in ('annex-externaltype', 'annex-webdav'):\n            if self.config.has_option(sec, opt):\n                return True\n        return False\n\n    @borrowkwargs(GitRepo)\n    def get_remotes(self,\n                    with_urls_only=False,\n                    exclude_special_remotes=False):\n        \"\"\"Get known (special-) remotes of the repository\n\n        Parameters\n        ----------\n        exclude_special_remotes: bool, optional\n          if True, don't return annex special remotes\n\n        Returns\n        -------\n        remotes : list of str\n          List of names of the remotes\n        \"\"\"\n        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)\n\n        if exclude_special_remotes:\n            return [\n                remote for remote in remotes\n                if not self.is_special_annex_remote(remote, check_if_known=False)\n            ]\n        else:\n            return remotes\n\n    def get_special_remotes(self):\n        \"\"\"Get info about all known (not just enabled) special remotes.\n\n        Returns\n        -------\n        dict\n          Keys are special remore UUIDs, values are dicts with arguments\n          for `git-annex enableremote`. This includes at least the 'type'\n          and 'name' of a special remote. Each type of special remote\n          may require addition arguments that will be available in the\n          respective dictionary.\n        \"\"\"\n        try:\n            stdout, stderr = self._git_custom_command(\n                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],\n                expect_fail=True)\n        except CommandError as e:\n            if 'Not a valid object name git-annex:remote.log' in e.stderr:\n                # no special remotes configures\n                return {}\n            else:\n                # some unforseen error\n                raise e\n        argspec = re.compile(r'^([^=]*)=(.*)$')\n        srs = {}\n        for line in stdout.splitlines():\n            # be precise and split by spaces\n            fields = line.split(' ')\n            # special remote UUID\n            sr_id = fields[0]\n            # the rest are config args for enableremote\n            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])\n            srs[sr_id] = sr_info\n        return srs\n\n    def __repr__(self):\n        return \"<AnnexRepo path=%s (%s)>\" % (self.path, type(self))\n\n    def _run_annex_command(self, annex_cmd,\n                           git_options=None, annex_options=None,\n                           backend=None, jobs=None,\n                           files=None,\n                           **kwargs):\n        \"\"\"Helper to run actual git-annex calls\n\n        Unifies annex command calls.\n\n        Parameters\n        ----------\n        annex_cmd: str\n            the actual git-annex command, like 'init' or 'add'\n        git_options: list of str\n            options to be passed to git\n        annex_options: list of str\n            options to be passed to the git-annex command\n        backend: str\n            backend to be used by this command; Currently this can also be\n            achieved by having an item '--backend=XXX' in annex_options.\n            This may change.\n        jobs : int\n        files: list, optional\n            If command passes list of files\n        **kwargs\n            these are passed as additional kwargs to datalad.cmd.Runner.run()\n\n        Raises\n        ------\n        CommandNotAvailableError\n            if an annex command call returns \"unknown command\"\n        \"\"\"\n        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []\n        backend = ['--backend=%s' % backend] if backend else []\n\n        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS\n        annex_options = annex_options[:] if annex_options else []\n        if self._annex_common_options:\n            annex_options = self._annex_common_options + annex_options\n\n        if not self.always_commit:\n            git_options += ['-c', 'annex.alwayscommit=false']\n\n        if git_options:\n            cmd_list = ['git'] + git_options + ['annex']\n        else:\n            cmd_list = ['git-annex']\n        if jobs:\n            annex_options += ['-J%d' % jobs]\n\n        cmd_list += [annex_cmd] + backend + debug + annex_options\n\n        env = kwargs.pop(\"env\", None)\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        if files:\n            cmd_list += ['--'] + files\n        try:\n            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)\n        except CommandError as e:\n            if e.stderr and \"git-annex: Unknown command '%s'\" % annex_cmd in e.stderr:\n                raise CommandNotAvailableError(str(cmd_list),\n                                               \"Unknown command:\"\n                                               \" 'git-annex %s'\" % annex_cmd,\n                                               e.code, e.stdout, e.stderr)\n            else:\n                raise e\n\n    def _run_simple_annex_command(self, *args, **kwargs):\n        \"\"\"Run an annex command and return its output, of which expect 1 line\n\n        Just a little helper to interact with basic annex commands and process\n        their output while ignoring some messages\n\n        Parameters\n        ----------\n        **kwargs: all passed into _run\n        \"\"\"\n        out, err = self._run_annex_command(\n            *args, **kwargs\n        )\n        lines = out.rstrip('\\n').splitlines()\n        # ignore some lines which might appear on a fresh clone\n        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/\n        lines_ = [\n            l for l in lines\n            if not re.search(\n                '\\((merging .* into git-annex|recording state ).*\\.\\.\\.\\)', l\n            )\n        ]\n        assert(len(lines_) <= 1)\n        return lines_[0] if lines_ else None\n\n    def _is_direct_mode_from_config(self):\n        \"\"\"Figure out if in direct mode from the git config.\n\n        Since relies on reading config, expensive to be used often\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # If .git/config lacks an entry \"direct\",\n        # it's actually indirect mode.\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"direct\", False)\n\n    def is_direct_mode(self):\n        \"\"\"Return True if annex is in direct mode\n\n        Returns\n        -------\n        True if in direct mode, False otherwise.\n        \"\"\"\n        # TEMP: Disable lazy loading and make sure to read from file every time\n        # instead, since we might have several instances pointing to the very\n        # same repo atm. TODO: We can remove that, right?\n        self.repo.config_reader()._is_initialized = False\n        self.repo.config_reader().read()\n        self._direct_mode = None\n\n        if self._direct_mode is None:\n            # we need to figure it out\n            self._direct_mode = self._is_direct_mode_from_config()\n        return self._direct_mode\n\n    def is_crippled_fs(self):\n        \"\"\"Return True if git-annex considers current filesystem 'crippled'.\n\n        Returns\n        -------\n        True if on crippled filesystem, False otherwise\n        \"\"\"\n\n        self.config.reload()\n        return self.config.getbool(\"annex\", \"crippledfilesystem\", False)\n\n    def set_direct_mode(self, enable_direct_mode=True):\n        \"\"\"Switch to direct or indirect mode\n\n        Parameters\n        ----------\n        enable_direct_mode: bool\n            True means switch to direct mode,\n            False switches to indirect mode\n\n        Raises\n        ------\n        CommandNotAvailableError\n            in case you try to switch to indirect mode on a crippled filesystem\n        \"\"\"\n        if self.is_crippled_fs() and not enable_direct_mode:\n            raise CommandNotAvailableError(\n                cmd=\"git-annex indirect\",\n                msg=\"Can't switch to indirect mode on that filesystem.\")\n\n        self._run_annex_command('direct' if enable_direct_mode else 'indirect',\n                                expect_stderr=True)\n        self.config.reload()\n\n        # For paranoid we will just re-request\n        self._direct_mode = None\n        assert(self.is_direct_mode() == enable_direct_mode)\n\n        if self.is_direct_mode():\n            # adjust git options for plain git calls on this repo:\n            # Note: Not sure yet, whether this solves the issue entirely or we\n            # still need 'annex proxy' in some cases ...\n\n            lgr.debug(\"detected git version: %s\" % external_versions['cmd:git'])\n\n            if external_versions['cmd:git'] >= '2.9.0':\n                # workaround for git 2.9.0, which for some reason ignores the\n                # per-call config \"-c core.bare=False\", but respects the value\n                # if it is set in .git/config\n                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True\n\n            # TEMP: nevertheless use this option to inject it into gitpython\n            # TODO: Solve it and change to \"elif\"\n            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part I:\n                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])\n            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:\n                # standard direct mode procedure part II:\n                self._GIT_COMMON_OPTIONS.append('--work-tree=.')\n\n    def _git_custom_command(self, *args, **kwargs):\n\n        if self.GIT_DIRECT_MODE_PROXY:\n            proxy_str = \"git annex proxy -- \"\n            proxy_list = ['git', 'annex', 'proxy', '--']\n            cmd = kwargs.pop(\"cmd_str\", None)\n            if not cmd:\n                cmd = args[1]\n            assert(cmd is not None)\n\n            if isinstance(cmd, string_types):\n                cmd = proxy_str + cmd\n            else:\n                cmd = proxy_list + cmd\n\n            args = (args[0], cmd) + args[2:]\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:\n            old = self.config.get('core.bare')\n            lgr.debug(\"old config: %s(%s)\" % (old, type(old)))\n            if old is not False:\n                self.config.set('core.bare', 'False', where='local')\n\n            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n            if old is None:\n                self.config.unset('core.bare', where='local')\n            elif old:\n                self.config.set('core.bare', old, where='local')\n            return out, err\n\n        else:\n            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)\n\n    def _init(self, version=None, description=None):\n        \"\"\"Initializes an annex repository.\n\n        Note: This is intended for private use in this class by now.\n        If you have an object of this class already,\n        there shouldn't be a need to 'init' again.\n\n        \"\"\"\n        # MIH: this function is required for re-initing repos. The logic\n        # in the constructor is rather convoluted and doesn't acknowledge\n        # the case of a perfectly healthy annex that just needs a new\n        # description\n        # will keep leading underscore in the name for know, but this is\n        # not private\n        # TODO: provide git and git-annex options.\n        # TODO: Document (or implement respectively) behaviour in special cases\n        # like direct mode (if it's different), not existing paths, etc.\n        opts = []\n        if description is not None:\n            opts += [description]\n        if version is not None:\n            opts += ['--version', '{0}'.format(version)]\n        if not len(opts):\n            opts = None\n\n        # Note: git-annex-init kills a possible tracking branch for\n        # 'annex/direct/my_branch', if we just cloned from a repo in direct\n        # mode. We want to preserve the information about the tracking branch,\n        # as if the source repo wasn't in direct mode.\n        # Note 2: Actually we do it for all 'managed branches'. This might turn\n        # out to not be necessary\n        sections_to_preserve = [\"branch.{}\".format(branch)\n                                for branch in self.get_branches()\n                                if self.is_managed_branch(branch)\n                                and \"branch.{}\".format(branch) in\n                                self.config.sections()]\n        for sct in sections_to_preserve:\n            orig_branch = sct[7:]\n            new_branch = self.get_corresponding_branch(orig_branch)\n            new_section = \"branch.{}\".format(new_branch)\n            for opt in self.config.options(sct):\n                orig_value = self.config.get_value(sct, opt)\n                new_value = orig_value.replace(orig_branch, new_branch)\n                self.config.add(var=new_section + \".\" + opt,\n                                value=new_value,\n                                where='local',\n                                reload=False)\n        self._run_annex_command(\n            'init',\n            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,\n            annex_options=opts)\n        # TODO: When to expect stderr?\n        # on crippled filesystem for example (think so)?\n        self.config.reload()\n\n    @normalize_paths\n    def get(self, files, remote=None, options=None, jobs=None, key=False):\n        \"\"\"Get the actual content of files\n\n        Parameters\n        ----------\n        files : list of str\n            paths to get\n        remote : str, optional\n            from which remote to fetch content\n        options : list of str, optional\n            commandline options for the git annex get command\n        jobs : int or None, optional\n            how many jobs to run in parallel (passed to git-annex call).\n            If not specified (None), then\n        key : bool, optional\n            If provided file value is actually a key\n\n        Returns\n        -------\n        files : list of dict\n        \"\"\"\n        options = options[:] if options else []\n\n        if remote:\n            if remote not in self.get_remotes():\n                raise RemoteNotAvailableError(\n                    remote=remote,\n                    cmd=\"get\",\n                    msg=\"Remote is not known. Known are: %s\"\n                    % (self.get_remotes(),)\n                )\n            options += ['--from', remote]\n\n        # analyze provided files to decide which actually are needed to be\n        # fetched\n\n        if not key:\n            expected_downloads, fetch_files = self._get_expected_files(\n                files, ['--not', '--in', 'here'])\n        else:\n            fetch_files = files\n            assert len(files) == 1, \"When key=True only a single file be provided\"\n            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not fetch_files:\n            lgr.debug(\"No files found needing fetching.\")\n            return []\n\n        if len(fetch_files) != len(files):\n            lgr.debug(\"Actually getting %d files\", len(fetch_files))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        # TODO: reproduce DK's bug on OSX, and either switch to\n        #  --batch mode (I don't think we have --progress support in long\n        #  alive batch processes ATM),\n        if key:\n            kwargs = {'opts': options + ['--key'] + files}\n        else:\n            kwargs = {'opts': options, 'files': files}\n        results = self._run_annex_command_json(\n            'get',\n            # TODO: eventually make use of --batch mode\n            jobs=jobs,\n            expected_entries=expected_downloads,\n            progress=True,\n            **kwargs\n        )\n        results_list = list(results)\n        # TODO:  should we here compare fetch_files against result_list\n        # and vomit an exception of incomplete download????\n        return results_list\n\n    def _get_expected_files(self, files, expr):\n        \"\"\"Given a list of files, figure out what to be downloaded\n\n        Parameters\n        ----------\n        files\n        expr: list\n          Expression to be passed into annex's find\n\n        Returns\n        -------\n        expected_files : dict\n          key -> size\n        fetch_files : list\n          files to be fetched\n        \"\"\"\n        lgr.debug(\"Determine what files match the query to work with\")\n        # Let's figure out first which files/keys and of what size to download\n        expected_files = {}\n        fetch_files = []\n        keys_seen = set()\n        unknown_sizes = []  # unused atm\n        # for now just record total size, and\n        for j in self._run_annex_command_json(\n                'find', opts=expr, files=files\n        ):\n            # TODO: some files might not even be here.  So in current fancy\n            # output reporting scheme we should then theoretically handle\n            # those cases here and say 'impossible' or something like that\n            if not j.get('success', True):\n                # TODO: I guess do something with yielding and filtering for\n                # what need to be done and what not\n                continue\n            key = j['key']\n            size = j.get('bytesize')\n            if key in keys_seen:\n                # multiple files could point to the same key.  no need to\n                # request multiple times\n                continue\n            keys_seen.add(key)\n            assert j['file']\n            fetch_files.append(j['file'])\n            if size and size.isdigit():\n                expected_files[key] = int(size)\n            else:\n                expected_files[key] = None\n                unknown_sizes.append(j['file'])\n        return expected_files, fetch_files\n\n    @normalize_paths\n    def add(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Add file(s) to the repository.\n\n        Parameters\n        ----------\n        files: list of str\n          list of paths to add to the annex\n        git: bool\n          if True, add to git instead of annex.\n        backend:\n        options:\n        update: bool\n          --update option for git-add. From git's manpage:\n           Update the index just where it already has an entry matching\n           <pathspec>. This removes as well as modifies index entries to match\n           the working tree, but adds no new files.\n\n           If no <pathspec> is given when --update option is used, all tracked\n           files in the entire working tree are updated (old versions of Git\n           used to limit the update to the current directory and its\n           subdirectories).\n\n           Note: Used only, if a call to git-add instead of git-annex-add is\n           performed\n\n        Returns\n        -------\n        list of dict\n        \"\"\"\n\n        return list(self.add_(\n            files, git=git, backend=backend, options=options, jobs=jobs,\n            git_options=git_options, annex_options=annex_options, update=update\n        ))\n\n    def add_(self, files, git=None, backend=None, options=None, jobs=None,\n            git_options=None, annex_options=None, update=False):\n        \"\"\"Like `add`, but returns a generator\"\"\"\n        if update and not git:\n            raise InsufficientArgumentsError(\"option 'update' requires 'git', too\")\n\n        if git_options:\n            # TODO: note that below we would use 'add with --dry-run\n            # so passed here options might need to be passed into it??\n            lgr.warning(\"add: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        # Note: As long as we support direct mode, one should not call\n        # super().add() directly. Once direct mode is gone, we might remove\n        # `git` parameter and call GitRepo's add() instead.\n\n        def _get_to_be_added_recs(paths):\n            \"\"\"Try to collect what actually is going to be added\n\n            This is used for progress information\n            \"\"\"\n\n            if self.is_direct_mode():\n                # we already know we can't use --dry-run\n                for r in self._process_git_get_output(\n                        linesep.join([\"'{}'\".format(p.encode('utf-8'))\n                        for p in paths])):\n                    yield r\n                    return\n            else:\n                # Note: if a path involves a submodule in direct mode, while we\n                # are not in direct mode at current level, we might still fail.\n                # Hence the except clause is still needed. However, this is\n                # unlikely, since direct mode usually should be used only, if it\n                # was enforced by FS and/or OS and therefore concerns the entire\n                # hierarchy.\n                _git_options = ['--dry-run', '-N', '--ignore-missing']\n                try:\n                    for r in super(AnnexRepo, self).add_(\n                            files, git_options=_git_options, update=update):\n                        yield r\n                        return\n                except CommandError as e:\n                    if AnnexRepo._is_annex_work_tree_message(e.stderr):\n                        lgr.warning(\n                            \"Known bug in direct mode.\"\n                            \"We can't use --dry-run when there are submodules in \"\n                            \"direct mode, because the internal call to git status \"\n                            \"fails. To be resolved by using (Dataset's) status \"\n                            \"instead of a git-add --dry-run altogether.\")\n                        # fake the return for now\n                        for r in self._process_git_get_output(\n                                linesep.join([\"'{}'\".format(f)\n                                for f in files])):\n                            yield r\n                            return\n                    else:\n                        # unexpected failure\n                        raise e\n\n        # Theoretically we could have done for git as well, if it could have\n        # been batched\n        # Call git annex add for any to have full control of either to go\n        # to git or to annex\n        # 1. Figure out what actually will be added\n        to_be_added_recs = _get_to_be_added_recs(files)\n        # collect their sizes for the progressbar\n        expected_additions = {\n            rec['file']: self.get_file_size(rec['file'])\n            for rec in to_be_added_recs\n        }\n\n        # if None -- leave it to annex to decide\n        if git is not None:\n            options += [\n                '-c',\n                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])\n            ]\n            if git:\n                # to maintain behaviour similar to git\n                options += ['--include-dotfiles']\n\n        if git and update:\n            # explicitly use git-add with --update instead of git-annex-add\n            # TODO: This might still need some work, when --update AND files\n            # are specified!\n            if self.is_direct_mode() and not files:\n                self.GIT_DIRECT_MODE_PROXY = True\n            try:\n                for r in super(AnnexRepo, self).add(\n                        files,\n                        git=True,\n                        git_options=git_options,\n                        update=update):\n                    yield r\n            finally:\n                if self.is_direct_mode() and not files:\n                    # don't accidentally cause other git calls to be done\n                    # via annex-proxy\n                    self.GIT_DIRECT_MODE_PROXY = False\n\n        else:\n            for r in self._run_annex_command_json(\n                    'add',\n                    opts=options,\n                    files=files,\n                    backend=backend,\n                    expect_fail=True,\n                    jobs=jobs,\n                    expected_entries=expected_additions,\n                    expect_stderr=True):\n                yield r\n\n    def proxy(self, git_cmd, **kwargs):\n        \"\"\"Use git-annex as a proxy to git\n\n        This is needed in case we are in direct mode, since there's no git\n        working tree, that git can handle.\n\n        Parameters\n        ----------\n        git_cmd: list of str\n            the actual git command\n        `**kwargs`: dict, optional\n            passed to _run_annex_command\n\n        Returns\n        -------\n        (stdout, stderr)\n            output of the command call\n        \"\"\"\n        # TODO: We probably don't need it anymore\n\n        if not self.is_direct_mode():\n            lgr.warning(\"proxy() called in indirect mode: %s\" % git_cmd)\n            raise CommandNotAvailableError(cmd=\"git annex proxy\",\n                                           msg=\"Proxy doesn't make sense\"\n                                               \" if not in direct mode.\")\n        return self._run_annex_command('proxy',\n                                       annex_options=['--'] + git_cmd,\n                                       **kwargs)\n\n    @normalize_paths\n    def get_file_key(self, files):\n        \"\"\"Get key of an annexed file.\n\n        Parameters\n        ----------\n        files: str or list\n            file(s) to look up\n\n        Returns\n        -------\n        str or list\n            keys used by git-annex for each of the files;\n            in case of a list an empty string is returned if there was no key\n            for that file\n        \"\"\"\n\n        if len(files) > 1:\n            return self._batched.get('lookupkey', path=self.path)(files)\n        else:\n            files = files[0]\n            # single file\n            # keep current implementation\n            # TODO: This should change, but involves more RF'ing and an\n            # alternative regarding FileNotInAnnexError\n            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages\n\n            try:\n                out, err = self._run_annex_command(\n                    'lookupkey',\n                    files=[files],\n                    expect_fail=True\n                )\n            except CommandError as e:\n                if e.code == 1:\n                    if not exists(opj(self.path, files)):\n                        raise IOError(e.code, \"File not found.\", files)\n                    # XXX you don't like me because I can be real slow!\n                    elif files in self.get_indexed_files():\n                        # if we got here, the file is present and in git,\n                        # but not in the annex\n                        raise FileInGitError(cmd=cmd_str,\n                                             msg=\"File not in annex, but git: %s\"\n                                                 % files,\n                                             filename=files)\n                    else:\n                        raise FileNotInAnnexError(cmd=cmd_str,\n                                                  msg=\"File not in annex: %s\"\n                                                      % files,\n                                                  filename=files)\n                else:\n                    # Not sure, whether or not this can actually happen\n                    raise e\n\n            entries = out.rstrip(linesep).splitlines()\n            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated\n            entries = list(filter(lambda x: not x.startswith('('), entries))\n            if len(entries) > 1:\n                lgr.warning(\"Got multiple entries in reply asking for a key of a file: %s\"\n                            % (str(entries)))\n            elif not entries:\n                raise FileNotInAnnexError(\"Could not get a key for a file(s) %s -- empty output\" % files)\n            return entries[0]\n\n    @normalize_paths\n    def lock(self, files, options=None):\n        \"\"\"undo unlock\n\n        Use  this to undo an unlock command if you don't want to modify the\n        files any longer, or have made modifications you want to discard.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n        \"\"\"\n\n        options = options[:] if options else []\n        self._run_annex_command('lock', annex_options=options, files=files)\n        # note: there seems to be no output by annex if success.\n\n    @normalize_paths\n    def unlock(self, files, options=None):\n        \"\"\"unlock files for modification\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unlocked files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        if self.is_direct_mode():\n\n            # TODO:\n            # If anything there should be a CommandNotAvailableError now:\n            lgr.debug(\"'%s' is in direct mode, \"\n                      \"'annex unlock' not available\", self)\n            lgr.warning(\"In direct mode there is no 'unlock'. However if \"\n                        \"the file's content is present, it is kind of \"\n                        \"unlocked. Therefore just checking whether this is \"\n                        \"the case.\")\n            # TODO/FIXME:\n            # Note: the following isn't exactly nice, if `files` is a dir.\n            # For a \"correct\" result we would need to report all files within\n            # potential dir(s) in `files`, that are annexed and have content.\n            # Also note, that even now files in git might be reported \"unlocked\",\n            # since they have content. This might be a confusing result.\n            # On the other hand, this is solved on the level of Dataset.unlock\n            # by annotating those paths 'notneeded' beforehand.\n            return [f for f in files if self.file_has_content(f)]\n\n        else:\n\n            # TODO: catch and parse output if failed (missing content ...)\n            std_out, std_err = \\\n                self._run_annex_command(\n                    'unlock', annex_options=options, files=files\n                )\n\n            return [line.split()[1]\n                    for line in std_out.splitlines()\n                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']\n\n    def adjust(self, options=None):\n        \"\"\"enter an adjusted branch\n\n        This command is only available in a v6 git-annex repository.\n\n        Parameters\n        ----------\n        options: list of str\n          currently requires '--unlock' or '--fix';\n          default: --unlock\n        \"\"\"\n        # TODO: Do we want to catch the case that\n        # \"adjusted/<current_branch_name>(unlocked)\" already exists and\n        # just check it out? Or fail like annex itself does?\n\n        # version check:\n        if not self.config.get(\"annex.version\") == '6':\n            raise CommandNotAvailableError(cmd='git annex adjust',\n                                           msg='git-annex-adjust requires a '\n                                               'version 6 repository')\n\n        options = options[:] if options else to_options(unlock=True)\n        self._run_annex_command('adjust', annex_options=options)\n\n    @normalize_paths\n    def unannex(self, files, options=None):\n        \"\"\"undo accidental add command\n\n        Use this to undo an accidental git annex add command. Note that for\n        safety, the content of the file remains in the annex, until you use git\n        annex unused and git annex dropunused.\n\n        Parameters\n        ----------\n        files: list of str\n        options: list of str\n\n        Returns\n        -------\n        list of str\n          successfully unannexed files\n        \"\"\"\n\n        options = options[:] if options else []\n\n        std_out, std_err = self._run_annex_command(\n            'unannex', annex_options=options, files=files\n        )\n        return [line.split()[1] for line in std_out.splitlines()\n                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']\n\n    @normalize_paths(map_filenames_back=True)\n    def find(self, files, batch=False):\n        \"\"\"Run `git annex find` on file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to find under annex\n        batch: bool, optional\n            initiate or continue with a batched run of annex find, instead of just\n            calling a single git annex find command. If any items in `files`\n            are directories, this value is treated as False.\n\n        Returns\n        -------\n        A dictionary the maps each item in `files` to its `git annex find`\n        result. Items without a successful result will be an empty string, and\n        multi-item results (which can occur for if `files` includes a\n        directory) will be returned as a list.\n        \"\"\"\n        objects = {}\n        # Ignore batch=True if any path is a directory because `git annex find\n        # --batch` always returns an empty string for directories.\n        if batch and not any(isdir(opj(self.path, f)) for f in files):\n            find = self._batched.get('find', json=True, path=self.path)\n            objects = {f: json_out.get(\"file\", \"\")\n                       for f, json_out in zip(files, find(files))}\n        else:\n            for f in files:\n                try:\n                    obj, er = self._run_annex_command(\n                        'find', files=[f],\n                        annex_options=[\"--print0\"],\n                        expect_fail=True\n                    )\n                    items = obj.rstrip(\"\\0\").split(\"\\0\")\n                    objects[f] = items[0] if len(items) == 1 else items\n                except CommandError:\n                    objects[f] = ''\n\n        return objects\n\n    def _check_files(self, fn, quick_fn, files, allow_quick, batch):\n        # Helper that isolates the common logic in `file_has_content` and\n        # `is_under_annex`. `fn` is the annex command used to do the check, and\n        # `quick_fn` is the non-annex variant.\n        is_v6 = self.config.get(\"annex.version\") == \"6\"\n        if is_v6 or self.is_direct_mode() or batch or not allow_quick:\n            # We're only concerned about modified files in V6 mode. In V5\n            # `find` returns an empty string for unlocked files, and in direct\n            # mode everything looks modified, so we don't even bother.\n            modified = self.get_changed_files() if is_v6 else []\n            annex_res = fn(files, normalize_paths=False, batch=batch)\n            return [bool(annex_res.get(f) and\n                         not (is_v6 and normpath(f) in modified))\n                    for f in files]\n        else:  # ad-hoc check which should be faster than call into annex\n            return [quick_fn(f) for f in files]\n\n    @normalize_paths\n    def file_has_content(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files have their content present under annex.\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being actually present.\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file has content locally\n        \"\"\"\n        # TODO: Also provide option to look for key instead of path\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            if islink(filepath):                    # if symlink\n                # find abspath of node pointed to by symlink\n                target_path = realpath(filepath)  # realpath OK\n                # TODO: checks for being not outside of this repository\n                # Note: ben removed '.git/' from '.git/annex/objects',\n                # since it is not true for submodules, whose '.git' is a\n                # symlink and being resolved to some\n                # '.git/modules/.../annex/objects'\n                return exists(target_path) and 'annex/objects' in str(target_path)\n            return False\n\n        return self._check_files(self.find, quick_check,\n                                 files, allow_quick, batch)\n\n    @normalize_paths\n    def is_under_annex(self, files, allow_quick=True, batch=False):\n        \"\"\"Check whether files are under annex control\n\n        Parameters\n        ----------\n        files: list of str\n            file(s) to check for being under annex\n        allow_quick: bool, optional\n            allow quick check, based on having a symlink into .git/annex/objects.\n            Works only in non-direct mode (TODO: thin mode)\n\n        Returns\n        -------\n        list of bool\n            For each input file states either file is under annex\n        \"\"\"\n        # theoretically in direct mode files without content would also be\n        # broken symlinks on the FSs which support it, but that would complicate\n        # the matters\n\n        # This is an ugly hack to prevent files from being treated as\n        # remotes by `git annex info`. See annex's `nameToUUID'`.\n        files = [opj(curdir, f) for f in files]\n\n        def check(files, **kwargs):\n            # Filter out directories because it doesn't make sense to ask if\n            # they are under annex control and `info` can only handle\n            # non-directories.\n            return self.info([f for f in files if not isdir(f)],\n                             fast=True, **kwargs)\n\n        def quick_check(filename):\n            filepath = opj(self.path, filename)\n            # todo checks for being not outside of this repository\n            # Note: ben removed '.git/' from '.git/annex/objects',\n            # since it is not true for submodules, whose '.git' is a\n            # symlink and being resolved to some\n            # '.git/modules/.../annex/objects'\n            return (\n                islink(filepath)\n                and 'annex/objects' in str(realpath(filepath))  # realpath OK\n            )\n\n        return self._check_files(check, quick_check,\n                                 files, allow_quick, batch)\n\n    def init_remote(self, name, options):\n        \"\"\"Creates a new special remote\n\n        Parameters\n        ----------\n        name: str\n            name of the special remote\n        \"\"\"\n        # TODO: figure out consistent way for passing options + document\n\n        self._run_annex_command('initremote', annex_options=[name] + options)\n        self.config.reload()\n\n    def enable_remote(self, name, env=None):\n        \"\"\"Enables use of an existing special remote\n\n        Parameters\n        ----------\n        name: str\n            name, the special remote was created with\n        \"\"\"\n\n        try:\n            self._run_annex_command(\n                'enableremote',\n                annex_options=[name],\n                expect_fail=True,\n                log_stderr=True,\n                env=env)\n        except CommandError as e:\n            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):\n                raise AccessDeniedError(e.stderr)\n            elif 'FailedConnectionException' in e.stderr:\n                raise AccessFailedError(e.stderr)\n            else:\n                raise e\n        self.config.reload()\n\n    def merge_annex(self, remote=None):\n        \"\"\"Merge git-annex branch\n\n        Merely calls `sync` with the appropriate arguments.\n\n        Parameters\n        ----------\n        remote: str, optional\n          Name of a remote to be \"merged\".\n        \"\"\"\n        self.sync(\n            remotes=remote, push=False, pull=False, commit=False, content=False,\n            all=False)\n\n    def sync(self, remotes=None, push=True, pull=True, commit=True,\n             content=False, all=False, fast=False):\n        \"\"\"Synchronize local repository with remotes\n\n        Use  this  command  when you want to synchronize the local repository\n        with one or more of its remotes. You can specify the remotes (or\n        remote groups) to sync with by name; the default if none are specified\n        is to sync with all remotes.\n\n        Parameters\n        ----------\n        remotes: str, list(str), optional\n          Name of one or more remotes to be sync'ed.\n        push : bool\n          By default, git pushes to remotes.\n        pull : bool\n          By default, git pulls from remotes\n        commit : bool\n          A commit is done by default. Disable to avoid  committing local\n          changes.\n        content : bool\n          Normally, syncing does not transfer the contents of annexed\n          files.  This option causes the content of files in the work tree\n          to also be uploaded and downloaded as necessary.\n        all : bool\n          This option, when combined with `content`, makes all available\n          versions of all files be synced, when preferred content settings\n          allow\n        fast : bool\n          Only sync with the remotes with the lowest annex-cost value\n          configured\n        \"\"\"\n        # In direct mode annex-sync fails, if commit=True\n        # apparently sync is calling git status internally, which then fails\n        # in the submodule. (As we already know)\n        # stdout:\n        # commit  add second ok\n        # (recording state in git...)\n        #\n        # failed\n        # (recording state in git...)\n        #\n        # stderr:\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n        # fatal: This operation must be run in a work tree\n        # fatal: 'git status --porcelain' failed in submodule submod\n        # git-annex: user error (xargs [\"-0\",\"git\",\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pathspecs\",\"add\",\"-f\"] exited 123)\n\n        # TODO: Workaround\n\n        args = []\n        args.extend(to_options(push=push, no_push=not push,\n                               # means: '--push' if push else '--no-push'\n                               pull=pull, no_pull=not pull,\n                               commit=commit, no_commit=not commit,\n                               content=content, no_content=not content,\n                               all=all,\n                               fast=fast))\n        args.extend(assure_list(remotes))\n        self._run_annex_command('sync', annex_options=args)\n\n    @normalize_path\n    def add_url_to_file(self, file_, url, options=None, backend=None,\n                        batch=False, git_options=None, annex_options=None,\n                        unlink_existing=False):\n        \"\"\"Add file from url to the annex.\n\n        Downloads `file` from `url` and add it to the annex.\n        If annex knows `file` already,\n        records that it can be downloaded from `url`.\n\n        Note: Consider using the higher-level `download_url` instead.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n\n        options: list\n            options to the annex command\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex addurl, instead of just\n            calling a single git annex addurl command\n\n        unlink_existing: bool, optional\n            by default crashes if file already exists and is under git.\n            With this flag set to True would first remove it.\n\n        Returns\n        -------\n        dict\n          In batch mode only ATM returns dict representation of json output returned\n          by annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_url_to_file: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n        git_options = []\n        kwargs = dict(backend=backend)\n        if lexists(opj(self.path, file_)) and \\\n                unlink_existing and \\\n                not self.is_under_annex(file_):\n            # already under git, we can't addurl for under annex\n            lgr.warning(\n                \"File %s:%s is already under git, removing so it could possibly\"\n                \" be added under annex\", self, file_\n            )\n            unlink(opj(self.path, file_))\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching addurl call \"\n                          \"because fake dates are enabled\")\n            files_opt = '--file=%s' % file_\n            if PY2:\n                files_opt = assure_bytes(files_opt)\n            out_json = self._run_annex_command_json(\n                'addurl',\n                opts=options + [files_opt] + [url],\n                progress=True,\n                expected_entries={file_: None},\n                **kwargs\n            )\n            if len(out_json) != 1:\n                raise AssertionError(\n                    \"should always be a single-item list, Got: %s\"\n                    % str(out_json))\n            # Make the output's structure match bcmd's.\n            out_json = out_json[0]\n            # Don't capture stderr, since download progress provided by wget uses\n            # stderr.\n        else:\n            options += ['--with-files']\n            if backend:\n                options += ['--backend=%s' % backend]\n            # Initializes (if necessary) and obtains the batch process\n            bcmd = self._batched.get(\n                # Since backend will be critical for non-existing files\n                'addurl_to_file_backend:%s' % backend,\n                annex_cmd='addurl',\n                git_options=git_options,\n                annex_options=options,  # --raw ?\n                path=self.path,\n                json=True\n            )\n            try:\n                out_json = bcmd((url, file_))\n            except Exception as exc:\n                # if isinstance(exc, IOError):\n                #     raise\n                raise AnnexBatchCommandError(\n                    cmd=\"addurl\",\n                    msg=\"Adding url %s to file %s failed due to %s\" % (url, file_, exc_str(exc)))\n            assert(out_json['command'] == 'addurl')\n        if not out_json.get('success', False):\n            raise (AnnexBatchCommandError if batch else CommandError)(\n                    cmd=\"addurl\",\n                    msg=\"Error, annex reported failure for addurl (url='%s'): %s\"\n                    % (url, str(out_json)))\n        return out_json\n\n    def add_urls(self, urls, options=None, backend=None, cwd=None,\n                 jobs=None,\n                 git_options=None, annex_options=None):\n        \"\"\"Downloads each url to its own file, which is added to the annex.\n\n        Parameters\n        ----------\n        urls: list of str\n\n        options: list, optional\n            options to the annex command\n\n        cwd: string, optional\n            working directory from within which to invoke git-annex\n        \"\"\"\n\n        if git_options:\n            lgr.warning(\"add_urls: git_options not yet implemented. Ignored.\")\n\n        if annex_options:\n            lgr.warning(\"annex_options not yet implemented. Ignored.\")\n\n        options = options[:] if options else []\n\n        self._run_annex_command('addurl', annex_options=options + urls,\n                                backend=backend, log_online=True,\n                                log_stderr=False, cwd=cwd)\n        # Don't capture stderr, since download progress provided by wget uses\n        # stderr.\n\n        # currently simulating similar return value, assuming success\n        # for all files:\n        # TODO: Make return values consistent across both *Repo classes!\n        return [{u'file': f, u'success': True} for f in urls]\n\n    @normalize_path\n    def rm_url(self, file_, url):\n        \"\"\"Record that the file is no longer available at the url.\n\n        Parameters\n        ----------\n        file_: str\n\n        url: str\n        \"\"\"\n\n        self._run_annex_command('rmurl', files=[file_, url])\n\n    @normalize_path\n    def get_urls(self, file_, key=False, batch=False):\n        \"\"\"Get URLs for a file/key\n\n        Parameters\n        ----------\n        file_: str\n        key: bool, optional\n            Either provided files are actually annex keys\n        \"\"\"\n        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']\n\n    @normalize_paths\n    def drop(self, files, options=None, key=False, jobs=None):\n        \"\"\"Drops the content of annexed files from this repository.\n\n        Drops only if possible with respect to required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        files: list of str\n            paths to drop\n        options : list of str, optional\n            commandline options for the git annex drop command\n        jobs : int, optional\n            how many jobs to run in parallel (passed to git-annex call)\n\n        Returns\n        -------\n        list(JSON objects)\n          'success' item in each object indicates failure/success per file\n          path.\n        \"\"\"\n\n        # annex drop takes either files or options\n        # --all, --unused, --key, or --incomplete\n        # for now, most simple test; to be replaced by a more general solution\n        # (exception thrown by _run_annex_command)\n        if not files and \\\n                (not options or\n                 not any([o in options for o in\n                          [\"--all\", \"--unused\", \"--key\", \"--incomplete\"]])):\n            raise InsufficientArgumentsError(\"drop() requires at least to \"\n                                             \"specify 'files' or 'options'\")\n\n        options = assure_list(options)\n\n        if key:\n            # we can't drop multiple in 1 line, and there is no --batch yet, so\n            # one at a time\n            files = assure_list(files)\n            options = options + ['--key']\n            res = [\n                self._run_annex_command_json(\n                    'drop',\n                    opts=options + [k],\n                    jobs=jobs)\n                for k in files\n            ]\n            # `normalize_paths` ... magic, useful?\n            if len(files) == 1:\n                return res[0]\n            else:\n                return res\n        else:\n            return self._run_annex_command_json(\n                'drop',\n                opts=options,\n                files=files,\n                jobs=jobs)\n\n    def drop_key(self, keys, options=None, batch=False):\n        \"\"\"Drops the content of annexed files from this repository referenced by keys\n\n        Dangerous: it drops without checking for required minimal number of\n        available copies.\n\n        Parameters\n        ----------\n        keys: list of str, str\n\n        batch: bool, optional\n            initiate or continue with a batched run of annex dropkey, instead of just\n            calling a single git annex dropkey command\n        \"\"\"\n        keys = [keys] if isinstance(keys, string_types) else keys\n\n        options = options[:] if options else []\n        options += ['--force']\n        if not batch or self.fake_dates_enabled:\n            if batch:\n                lgr.debug(\"Not batching drop_key call \"\n                          \"because fake dates are enabled\")\n            json_objects = self._run_annex_command_json(\n                'dropkey', opts=options, files=keys, expect_stderr=True\n            )\n        else:\n            json_objects = self._batched.get(\n                'dropkey',\n                annex_options=options, json=True, path=self.path\n            )(keys)\n        for j in json_objects:\n            assert j.get('success', True)\n\n    # TODO: a dedicated unit-test\n    def _whereis_json_to_dict(self, j):\n        \"\"\"Convert json record returned by annex whereis --json to our dict representation for it\n        \"\"\"\n        assert (j.get('success', True) is True)\n        # process 'whereis' containing list of remotes\n        remotes = {remote['uuid']: {x: remote.get(x, None)\n                                    for x in ('description', 'here', 'urls')\n                                    }\n                   for remote in j.get('whereis')}\n        if self.WEB_UUID in remotes:\n            assert(remotes[self.WEB_UUID]['description'] == 'web')\n        return remotes\n\n    def _run_annex_command_json(self, command,\n                                opts=None,\n                                jobs=None,\n                                files=None,\n                                expected_entries=None,\n                                progress=False,\n                                **kwargs):\n        \"\"\"Run an annex command with --json and load output results into a tuple of dicts\n\n        Parameters\n        ----------\n        expected_entries : dict, optional\n          If provided `filename/key: size` dictionary, will be used to create\n          ProcessAnnexProgressIndicators to display progress\n        progress: bool, optional\n          Whether to request/handle --json-progress\n        \"\"\"\n        progress_indicators = None\n        try:\n            if expected_entries:\n                progress_indicators = ProcessAnnexProgressIndicators(\n                    expected=expected_entries\n                )\n                kwargs = kwargs.copy()\n                kwargs.update(dict(\n                    log_stdout=progress_indicators,\n                    log_stderr='offline',  # False, # to avoid lock down\n                    log_online=True\n                ))\n            # TODO: refactor to account for possible --batch ones\n            annex_options = ['--json']\n            if progress:\n                annex_options += ['--json-progress']\n\n            if jobs == 'auto':\n                jobs = N_AUTO_JOBS\n            if jobs and jobs != 1:\n                annex_options += ['-J%d' % jobs]\n            if opts:\n                # opts might be the '--key' which should go last\n                annex_options += opts\n\n            # TODO: RF to use --batch where possible instead of splitting\n            # into multiple invocations\n            if not files:\n                file_chunks = [[]]\n            else:\n                files = assure_list(files)\n                maxl = max(map(len, files))\n                chunk_size = max(\n                    1,  # should at least be 1. If blows then - not our fault\n                    (CMD_MAX_ARG\n                     - len(command)\n                     - sum((len(x) + 3) for x in annex_options)\n                     - 4   # for '--' below\n                     - 50  # for safety since we are to add more options\n                    // (maxl + 3)  # +3 for possible quotes and a space\n                    )\n                )\n                file_chunks = generate_chunks(files, chunk_size)\n            out, err = \"\", \"\"\n            for file_chunk in file_chunks:\n                out_, err_ = self._run_annex_command(\n                    command,\n                    annex_options=annex_options + ['--'] + file_chunk,\n                    **kwargs)\n                out += out_\n                err += err_\n        except CommandError as e:\n            # Note: A call might result in several 'failures', that can be or\n            # cannot be handled here. Detection of something, we can deal with,\n            # doesn't mean there's nothing else to deal with.\n\n            # OutOfSpaceError:\n            # Note:\n            # doesn't depend on anything in stdout. Therefore check this before\n            # dealing with stdout\n            out_of_space_re = re.search(\n                \"not enough free space, need (.*) more\", e.stderr\n            )\n            if out_of_space_re:\n                raise OutOfSpaceError(cmd=\"annex %s\" % command,\n                                      sizemore_msg=out_of_space_re.groups()[0])\n\n            # RemoteNotAvailableError:\n            remote_na_re = re.search(\n                \"there is no available git remote named \\\"(.*)\\\"\", e.stderr\n            )\n            if remote_na_re:\n                raise RemoteNotAvailableError(cmd=\"annex %s\" % command,\n                                              remote=remote_na_re.groups()[0])\n\n            # TEMP: Workaround for git-annex bug, where it reports success=True\n            # for annex add, while simultaneously complaining, that it is in\n            # a submodule:\n            # TODO: For now just reraise. But independently on this bug, it\n            # makes sense to have an exception for that case\n            in_subm_re = re.search(\n                \"fatal: Pathspec '(.*)' is in submodule '(.*)'\", e.stderr\n            )\n            if in_subm_re:\n                raise e\n\n            # Note: try to approach the covering of potential annex failures\n            # in a more general way:\n            # first check stdout:\n            if all([line.startswith('{') and line.endswith('}')\n                    for line in e.stdout.splitlines()]):\n                # we have the usual json output on stdout. Therefore we can\n                # probably return and don't need to raise; so get stdout\n                # for json loading:\n                out = e.stdout\n            else:\n                out = None\n\n            # Note: Workaround for not existing files as long as annex doesn't\n            # report it within JSON response:\n            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/\n            not_existing = [\n                line.split()[1] for line in e.stderr.splitlines()\n                if line.startswith('git-annex:') and\n                   line.endswith('not found')\n            ]\n            if not_existing:\n                if out is None:\n                    # we create the error reporting herein. If all files were\n                    # not found, there is nothing on stdout and we don't need\n                    # anything\n                    out = \"\"\n                if not out.endswith(linesep):\n                    out += linesep\n                out += linesep.join(\n                    ['{{\"command\": \"{cmd}\", \"file\": \"{path}\", '\n                     '\"note\": \"{note}\",'\n                     '\"success\":false}}'.format(\n                         cmd=command, path=f, note=\"not found\")\n                     for f in not_existing])\n\n            # Note: insert additional code here to analyse failure and possibly\n            # raise a custom exception\n\n            # if we didn't raise before, just depend on whether or not we seem\n            # to have some json to return. It should contain information on\n            # failure in keys 'success' and 'note'\n            # TODO: This is not entirely true. 'annex status' may return empty,\n            # while there was a 'fatal:...' in stderr, which should be a\n            # failure/exception\n            # Or if we had empty stdout but there was stderr\n            if out is None or (not out and e.stderr):\n                raise e\n            if e.stderr:\n                # else just warn about present errors\n                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x\n                lgr.warning(\n                    \"Running %s resulted in stderr output: %s\",\n                    command, shorten(e.stderr)\n                )\n        finally:\n            if progress_indicators:\n                progress_indicators.finish()\n\n        json_objects = (json_loads(line)\n                        for line in out.splitlines() if line.startswith('{'))\n        # protect against progress leakage\n        json_objects = [j for j in json_objects if 'byte-progress' not in j]\n        return json_objects\n\n    # TODO: reconsider having any magic at all and maybe just return a list/dict always\n    @normalize_paths\n    def whereis(self, files, output='uuids', key=False, options=None, batch=False):\n        \"\"\"Lists repositories that have actual content of file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n        output: {'descriptions', 'uuids', 'full'}, optional\n            If 'descriptions', a list of remotes descriptions returned is per\n            each file. If 'full', for each file a dictionary of all fields\n            is returned as returned by annex\n        key: bool, optional\n            Either provided files are actually annex keys\n        options: list, optional\n            Options to pass into git-annex call\n\n        Returns\n        -------\n        list of list of unicode  or dict\n            if output == 'descriptions', contains a list of descriptions of remotes\n            for each input file, describing the remote for each remote, which\n            was found by git-annex whereis, like::\n\n                u'me@mycomputer:~/where/my/repo/is [origin]' or\n                u'web' or\n                u'me@mycomputer:~/some/other/clone'\n\n            if output == 'uuids', returns a list of uuids.\n            if output == 'full', returns a dictionary with filenames as keys\n            and values a detailed record, e.g.::\n\n                {'00000000-0000-0000-0000-000000000001': {\n                  'description': 'web',\n                  'here': False,\n                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']\n                }}\n        \"\"\"\n        if batch:\n            lgr.warning(\"TODO: --batch mode for whereis.  Operating serially\")\n\n        OUTPUTS = {'descriptions', 'uuids', 'full'}\n        if output not in OUTPUTS:\n            raise ValueError(\n                \"Unknown value output=%r. Known are %s\"\n                % (output, ', '.join(map(repr, OUTPUTS)))\n            )\n\n        options = assure_list(options, copy=True)\n        if key:\n            kwargs = {'opts': options + [\"--key\"] + files}\n        else:\n            kwargs = {'files': files}\n\n        json_objects = self._run_annex_command_json('whereis', **kwargs)\n        if output in {'descriptions', 'uuids'}:\n            return [\n                [remote.get(output[:-1]) for remote in j.get('whereis')]\n                if j.get('success') else []\n                for j in json_objects\n            ]\n        elif output == 'full':\n            # TODO: we might want to optimize storage since many remotes entries will be the\n            # same so we could just reuse them instead of brewing copies\n            return {\n                j['key' if (key or '--all' in options) else 'file']\n                : self._whereis_json_to_dict(j)\n                for j in json_objects\n                if not j.get('key').endswith('.this-is-a-test-key')\n            }\n\n    # TODO:\n    # I think we should make interface cleaner and less ambigious for those annex\n    # commands which could operate on globs, files, and entire repositories, separating\n    # those out, e.g. annex_info_repo, annex_info_files at least.\n    # If we make our calling wrappers work without relying on invoking from repo topdir,\n    # then returned filenames would not need to be mapped, so we could easily work on dirs\n    # and globs.\n    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?\n    @normalize_paths(map_filenames_back=True)\n    def info(self, files, batch=False, fast=False):\n        \"\"\"Provide annex info for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n            files to look for\n\n        Returns\n        -------\n        dict\n          Info for each file\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        if not batch:\n            json_objects = self._run_annex_command_json(\n                'info', opts=options, files=files)\n        else:\n            json_objects = self._batched.get(\n                'info',\n                annex_options=options, json=True, path=self.path\n            )(files)\n\n        # Some aggressive checks. ATM info can be requested only per file\n        # json_objects is a generator, let's keep it that way\n        # assert(len(json_objects) == len(files))\n        # and that they all have 'file' equal to the passed one\n        out = {}\n        for j, f in zip(json_objects, files):\n            assert(j.pop('file') == f)\n            if not j['success']:\n                j = None\n            else:\n                assert(j.pop('success') is True)\n                # convert size to int\n                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None\n                # and pop the \"command\" field\n                j.pop(\"command\")\n            out[f] = j\n        return out\n\n    def repo_info(self, fast=False):\n        \"\"\"Provide annex info for the entire repository.\n\n        Returns\n        -------\n        dict\n          Info for the repository, with keys matching the ones returned by annex\n        \"\"\"\n\n        options = ['--bytes', '--fast'] if fast else ['--bytes']\n\n        json_records = list(self._run_annex_command_json('info', opts=options))\n        assert(len(json_records) == 1)\n\n        # TODO: we need to abstract/centralize conversion from annex fields\n        # For now just tune up few for immediate usability\n        info = json_records[0]\n        for k in info:\n            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):\n                size = info[k].split()[0]\n                if size.isdigit():\n                    info[k] = int(size)\n                else:\n                    lgr.debug(\"Size %r reported to be %s, setting to None\", k, size)\n                    info[k] = None\n        assert(info.pop('success'))\n        assert(info.pop('command') == 'info')\n        return info  # just as is for now\n\n    def get_annexed_files(self, with_content_only=False, patterns=None):\n        \"\"\"Get a list of files in annex\n\n        Parameters\n        ----------\n        with_content_only : bool, optional\n            Only list files whose content is present.\n        patterns : list, optional\n            Globs to pass to annex's `--include=`. Files that match any of\n            these will be returned (i.e., they'll be separated by `--or`).\n\n        Returns\n        -------\n        A list of file names\n        \"\"\"\n        if not patterns:\n            args = [] if with_content_only else ['--include', \"*\"]\n        else:\n            if len(patterns) == 1:\n                args = ['--include', patterns[0]]\n            else:\n                args = ['-(']\n                for pat in patterns[:-1]:\n                    args.extend(['--include', pat, \"--or\"])\n                args.extend(['--include', patterns[-1]])\n                args.append('-)')\n\n            if with_content_only:\n                args.extend(['--in', 'here'])\n        out, err = self._run_annex_command('find', annex_options=args)\n        # TODO: JSON\n        return out.splitlines()\n\n    def get_preferred_content(self, property, remote=None):\n        \"\"\"Get preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        remote : str, optional\n          If not specified (None), returns the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Either the setting is returned, or an empty string if there\n          is none.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.'])\n\n    def set_preferred_content(self, property, expr, remote=None):\n        \"\"\"Set preferred content configuration of a repository or remote\n\n        Parameters\n        ----------\n        property : {'wanted', 'required', 'group'}\n          Type of property to query\n        expr : str\n          Any expression or label supported by git-annex for the\n          given property.\n        remote : str, optional\n          If not specified (None), sets the property for the local\n          repository.\n\n        Returns\n        -------\n        str\n          Raw git-annex output in response to the set command.\n\n        Raises\n        ------\n        ValueError\n          If an unknown property label is given.\n\n        CommandError\n          If the annex call errors.\n        \"\"\"\n        if property not in ('wanted', 'required', 'group'):\n            raise ValueError(\n                'unknown preferred content property: {}'.format(property))\n        return self._run_simple_annex_command(\n            property,\n            annex_options=[remote or '.', expr])\n\n    def get_groupwanted(self, name):\n        \"\"\"Get `groupwanted` expression for a group `name`\n\n        Parameters\n        ----------\n        name : str\n           Name of the groupwanted group\n        \"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name]\n        )\n\n    def set_groupwanted(self, name, expr):\n        \"\"\"Set `expr` for the `name` groupwanted\"\"\"\n        return self._run_simple_annex_command(\n            'groupwanted', annex_options=[name, expr]\n        )\n\n    def precommit(self):\n        \"\"\"Perform pre-commit maintenance tasks, such as closing all batched annexes\n        since they might still need to flush their changes into index\n        \"\"\"\n        if self._batched is not None:\n            self._batched.close()\n        super(AnnexRepo, self).precommit()\n\n    @borrowdoc(GitRepo)\n    def commit(self, msg=None, options=None, _datalad_msg=False,\n               careless=True, files=None, proxy=False):\n        self.precommit()\n\n        if files:\n            files = assure_list(files)\n\n        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy\n        #       in direct mode. This is needed in very special cases, which\n        #       might go away once we figured out a better way. In any case, it\n        #       should turn into something that is automatically considered and\n        #       not done by the caller of this method.\n\n        if proxy:\n            if not self.is_direct_mode():\n                raise CommandNotAvailableError(\n                    cmd=\"git-annex-proxy\",\n                    msg=\"git-annex-proxy is available in direct mode only\")\n            else:\n                if _datalad_msg:\n                    msg = self._get_prefixed_commit_msg(msg)\n                if not msg:\n                    if options:\n                        if \"--allow-empty-message\" not in options:\n                            options.append(\"--allow-empty-message\")\n                    else:\n                        options = [\"--allow-empty-message\"]\n\n                # committing explicitly given paths in direct mode via proxy\n                # used to fail, because absolute paths are used. Using annex\n                # proxy this leads to an error (path outside repository)\n                if files:\n                    if options is None:\n                        options = []\n                    for i in range(len(files)):\n                        if isabs(files[i]):\n                            options.append(normpath(relpath(files[i],\n                                                            start=self.path)))\n                        else:\n                            options.append(files[i])\n                try:\n                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +\n                               (options if options else []),\n                               expect_stderr=True, expect_fail=True)\n                except CommandError as e:\n                    if 'nothing to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"nothing to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif 'no changes added to commit' in e.stdout or \\\n                            'nothing added to commit' in e.stdout:\n                        if careless:\n                            lgr.debug(\"no changes added to commit in {}. \"\n                                      \"Ignored.\".format(self))\n                        else:\n                            raise\n                    elif \"did not match any file(s) known to git.\" in e.stderr:\n                        # TODO:\n                        # Improve FileNotInXXXXError classes to better deal with\n                        # multiple files; Also consider PathOutsideRepositoryError\n                        raise FileNotInRepositoryError(cmd=e.cmd,\n                                                       msg=\"File(s) unknown to git\",\n                                                       code=e.code,\n                                                       filename=linesep.join(\n                                                    [l for l in e.stderr.splitlines()\n                                                     if l.startswith(\"pathspec\")]))\n                    else:\n                        raise\n        else:\n\n            # Note: See the note on `proxy` parameter at the top of this method.\n            #       Trying to automatically use git-annex-proxy, whenever we\n            #       fail to commit the usual way via options to git in direct\n            #       mode. In particular this can happen if sth was staged via\n            #       git-annex-proxy, which is needed for --update option for\n            #       example.\n\n            if files:\n                # Raise FileNotInRepositoryError if `files` aren't tracked.\n                try:\n                    super(AnnexRepo, self).commit(\n                        \"dryrun\", options=[\"--dry-run\", \"--no-status\"],\n                        files=files)\n                except CommandError as e:\n                    if not (self.is_direct_mode() and\n                            AnnexRepo._is_annex_work_tree_message(e.stderr)):\n                        raise\n                    # The git call may fail with\n                    #   fatal: this operation must be run in a work tree\n                    #   fatal: 'git status --porcelain=2' failed in submodule ...\n                    # But that error message doesn't matter for the purpose of\n                    # the \"file is tracked\" check. It won't be shown if there\n                    # is a pathspec error.\n                    lgr.debug(\"Ignoring commit --dry-run failure\")\n            try:\n                alt_index_file = None\n\n                direct_mode = self.is_direct_mode()\n                # we might need to avoid explicit paths\n                files_to_commit = None if direct_mode else files\n                if direct_mode and files:\n                    # In direct mode, if we commit file(s) they would get\n                    # committed directly into git ignoring possibly being\n                    # staged by annex.  So, if not all files are committed, and\n                    # assuming that what is to be committed is staged (!could be\n                    # wrong assumption), we need to prepare a custom index, and\n                    # commit without specifying any paths\n                    changed_files = {\n                        staged: set(self.get_changed_files(staged=staged))\n                        for staged in (True,) #  False}\n                    }\n                    files_set = {\n                        _normalize_path(self.path, f) if isabs(f) else f\n                        for f in files}\n                    files_notstaged = files_set.difference(changed_files[True])\n                    # Lazy .add('.') results in ALL (performance?) files listed\n                    # here even if they are not changed at all and could be ignored\n                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])\n                    # if files_changed_notstaged:\n                    #     # To be safe(r), let's just blow for now!\n                    #     # We could be smart(er) and do all the git or annex add\n                    #     # but let's see if we run into those cases first\n                    #     import pdb; pdb.set_trace()\n                    #     raise RuntimeError(\n                    #         \"To commit files in direct mode, please first git \"\n                    #         \"or git-annex add them first!  Files: %s\"\n                    #         % ', '.join(files_changed_notstaged)\n                    #     )\n                    # Files which were staged but not among files\n                    staged_not_to_commit = changed_files[True].difference(files_set)\n                    if staged_not_to_commit:\n                        # Need an alternative index_file\n                        with make_tempfile() as index_file:\n                            alt_index_file = index_file\n                            index_tree = self.repo.git.write_tree()\n                            self.repo.git.read_tree(index_tree,\n                                                    index_output=index_file)\n                            # Reset the files we are not to be committed\n                            self._git_custom_command(\n                                list(staged_not_to_commit),\n                                ['git', 'reset'],\n                                index_file=alt_index_file)\n\n                            super(AnnexRepo, self).commit(\n                                msg, options,\n                                _datalad_msg=_datalad_msg,\n                                careless=careless,\n                                index_file=alt_index_file)\n\n                    # in any case we will not specify files explicitly\n                    files_to_commit = None\n                if not alt_index_file:\n                    super(AnnexRepo, self).commit(msg, options,\n                                                  _datalad_msg=_datalad_msg,\n                                                  careless=careless,\n                                                  files=files_to_commit)\n            except CommandError as e:\n                # TODO: just remove this???\n                if self.is_direct_mode() and \\\n                    AnnexRepo._is_annex_work_tree_message(e.stderr):\n                    lgr.debug(\"Commit failed. \"\n                              \"Trying to commit via git-annex-proxy.\")\n                    self.commit(msg, options, _datalad_msg=_datalad_msg,\n                                careless=careless, files=files, proxy=True)\n                else:\n                    raise\n            finally:\n                if alt_index_file and os.path.exists(alt_index_file):\n                    unlink(alt_index_file)\n\n    @normalize_paths(match_return_type=False)\n    def remove(self, files, force=False, **kwargs):\n        \"\"\"Remove files from git/annex (works in direct mode as well)\n\n        Parameters\n        ----------\n        files\n        force: bool, optional\n        \"\"\"\n\n        # TODO: parameter 'force' unnecessary => kwargs / to_options\n        self.precommit()  # since might interfere\n\n        return super(AnnexRepo, self).remove(files, force=force,\n                                             normalize_paths=False,\n                                             **kwargs)\n\n    def get_contentlocation(self, key, batch=False):\n        \"\"\"Get location of the key content\n\n        Normally under .git/annex objects in indirect mode and within file\n        tree in direct mode.\n\n        Unfortunately there is no (easy) way to discriminate situations\n        when given key is simply incorrect (not known to annex) or its content\n        not currently present -- in both cases annex just silently exits with -1\n\n\n        Parameters\n        ----------\n        key: str\n            key\n        batch: bool, optional\n            initiate or continue with a batched run of annex contentlocation\n\n        Returns\n        -------\n        str\n            path relative to the top directory of the repository. If no content\n            is present, empty string is returned\n        \"\"\"\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('contentlocation',\n                                                   annex_options=[key],\n                                                   expect_fail=True)\n                return out.rstrip(linesep).splitlines()[0]\n            except CommandError:\n                return ''\n        else:\n            return self._batched.get('contentlocation', path=self.path)(key)\n\n    @normalize_paths(serialize=True)\n    def is_available(self, file_, remote=None, key=False, batch=False):\n        \"\"\"Check if file or key is available (from a remote)\n\n        In case if key or remote is misspecified, it wouldn't fail but just keep\n        returning False, although possibly also complaining out loud ;)\n\n        Parameters\n        ----------\n        file_: str\n            Filename or a key\n        remote: str, optional\n            Remote which to check.  If None, possibly multiple remotes are checked\n            before positive result is reported\n        key: bool, optional\n            Either provided files are actually annex keys\n        batch: bool, optional\n            Initiate or continue with a batched run of annex checkpresentkey\n\n        Returns\n        -------\n        bool\n            with True indicating that file/key is available from (the) remote\n        \"\"\"\n\n        if key:\n            key_ = file_\n        else:\n            key_ = self.get_file_key(file_)  # ?, batch=batch\n\n        annex_input = (key_,) if not remote else (key_, remote)\n\n        if not batch:\n            try:\n                out, err = self._run_annex_command('checkpresentkey',\n                                                   annex_options=list(annex_input),\n                                                   expect_fail=True)\n                assert(not out)\n                return True\n            except CommandError:\n                return False\n        else:\n            annex_cmd = [\"checkpresentkey\"] + ([remote] if remote else [])\n            out = self._batched.get(\n                ':'.join(annex_cmd), annex_cmd,\n                path=self.path)(key_)\n            try:\n                return {\n                    # happens on travis in direct/heavy-debug mode, that process\n                    # exits and closes stdout (upon unknown key) before we could\n                    # read it, so we get None as the stdout.\n                    # see https://github.com/datalad/datalad/issues/2330\n                    # but it is associated with an unknown key, and for consistency\n                    # we report False there too, as to ''\n                    None: False,\n                    '': False,  # when remote is misspecified ... stderr carries the msg\n                    '0': False,\n                    '1': True,\n                }[out]\n            except KeyError:\n                raise ValueError(\n                    \"Received output %r from annex, whenever expect 0 or 1\" % out\n                )\n\n    @normalize_paths(match_return_type=False)\n    def _annex_custom_command(\n            self, files, cmd_str, log_stdout=True, log_stderr=True,\n            log_online=False, expect_stderr=False, cwd=None, env=None,\n            shell=None, expect_fail=False):\n        \"\"\"Allows for calling arbitrary commands.\n\n        Helper for developing purposes, i.e. to quickly implement git-annex\n        commands for proof of concept.\n\n        Parameters\n        ----------\n        files: list of files\n        cmd_str: str\n            arbitrary command str. `files` is appended to that string.\n\n        Returns\n        -------\n        stdout, stderr\n        \"\"\"\n        cmd = shlex.split(cmd_str + \" \" + \" \".join(files), posix=not on_windows) \\\n            if isinstance(cmd_str, string_types) \\\n            else cmd_str + files\n\n        if self.fake_dates_enabled:\n            env = self.add_fake_dates(env)\n\n        return self.cmd_call_wrapper.run(\n            cmd,\n            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,\n            expect_stderr=expect_stderr,\n            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)\n\n    @normalize_paths\n    def migrate_backend(self, files, backend=None):\n        \"\"\"Changes the backend used for `file`.\n\n        The backend used for the key-value of `files`. Only files currently\n        present are migrated.\n        Note: There will be no notification if migrating fails due to the\n        absence of a file's content!\n\n        Parameters\n        ----------\n        files: list\n            files to migrate.\n        backend: str\n            specify the backend to migrate to. If none is given, the\n            default backend of this instance will be used.\n        \"\"\"\n\n        if self.is_direct_mode():\n            raise CommandNotAvailableError(\n                'git-annex migrate',\n                \"Command 'migrate' is not available in direct mode.\")\n        self._run_annex_command('migrate',\n                                annex_options=files,\n                                backend=backend)\n\n    @normalize_paths\n    def get_file_backend(self, files):\n        \"\"\"Get the backend currently used for file(s).\n\n        Parameters\n        ----------\n        files: list of str\n\n        Returns\n        -------\n        list of str\n            For each file in input list indicates the used backend by a str\n            like \"SHA256E\" or \"MD5\".\n        \"\"\"\n\n        return [self.get_file_key(f).split('-')[0] for f in files]\n\n    @property\n    def default_backends(self):\n        self.config.reload()\n        backends = self.config.get(\"annex.backends\", default=None)\n        if backends:\n            return backends.split()\n        else:\n            return None\n\n    def fsck(self):\n        self._run_annex_command('fsck')\n\n    # TODO: we probably need to override get_file_content, since it returns the\n    # symlink's target instead of the actual content.\n\n    # We need --auto and --fast having exposed  TODO\n    @normalize_paths(match_return_type=False)  # get a list even in case of a single item\n    def copy_to(self, files, remote, options=None, jobs=None):\n        \"\"\"Copy the actual content of `files` to `remote`\n\n        Parameters\n        ----------\n        files: str or list of str\n            path(s) to copy\n        remote: str\n            name of remote to copy `files` to\n\n        Returns\n        -------\n        list of str\n           files successfully copied\n        \"\"\"\n\n        # find --in here --not --in remote\n        # TODO: full support of annex copy options would lead to `files` being\n        # optional. This means to check for whether files or certain options are\n        # given and fail or just pass everything as is and try to figure out,\n        # what was going on when catching CommandError\n\n        if remote not in self.get_remotes():\n            raise ValueError(\"Unknown remote '{0}'.\".format(remote))\n\n        options = options[:] if options else []\n\n        # Note:\n        # In case of single path, 'annex copy' will fail, if it cannot copy it.\n        # With multiple files, annex will just skip the ones, it cannot deal\n        # with. We'll do the same and report back what was successful\n        # (see return value).\n        # Therefore raise telling exceptions before even calling annex:\n        if len(files) == 1:\n            if not isdir(files[0]):\n                self.get_file_key(files[0])\n\n        # TODO: RF -- logic is duplicated with get() -- the only difference\n        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?\n        if '--key' not in options:\n            expected_copys, copy_files = self._get_expected_files(\n                files, ['--in', 'here', '--not', '--in', remote])\n        else:\n            copy_files = files\n            assert(len(files) == 1)\n            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}\n\n        if not copy_files:\n            lgr.debug(\"No files found needing copying.\")\n            return []\n\n        if len(copy_files) != len(files):\n            lgr.debug(\"Actually copying %d files\", len(copy_files))\n\n        annex_options = ['--to=%s' % remote]\n        if options:\n            annex_options.extend(shlex.split(options))\n\n        # TODO: provide more meaningful message (possibly aggregating 'note'\n        #  from annex failed ones\n        results = self._run_annex_command_json(\n            'copy',\n            opts=annex_options,\n            files=files,  # copy_files,\n            jobs=jobs,\n            expected_entries=expected_copys,\n            progress=True\n            #log_stdout=True, log_stderr=not log_online,\n            #log_online=log_online, expect_stderr=True\n        )\n        results_list = list(results)\n        # XXX this is the only logic different ATM from get\n        # check if any transfer failed since then we should just raise an Exception\n        # for now to guarantee consistent behavior with non--json output\n        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456\n        from operator import itemgetter\n        failed_copies = [e['file'] for e in results_list if not e['success']]\n        good_copies = [\n            e['file'] for e in results_list\n            if e['success'] and\n               e.get('note', '').startswith('to ')  # transfer did happen\n        ]\n        if failed_copies:\n            # TODO: RF for new fancy scheme of outputs reporting\n            raise IncompleteResultsError(\n                results=good_copies, failed=failed_copies,\n                msg=\"Failed to copy %d file(s)\" % len(failed_copies))\n        return good_copies\n\n    @property\n    def uuid(self):\n        \"\"\"Annex UUID\n\n        Returns\n        -------\n        str\n          Returns a the annex UUID, if there is any, or `None` otherwise.\n        \"\"\"\n        if not self._uuid:\n            if not self.repo:\n                return None\n            self._uuid = self.config.get('annex.uuid', default=None)\n        return self._uuid\n\n    def get_description(self, uuid=None):\n        \"\"\"Get annex repository description\n\n        Parameters\n        ----------\n        uuid : str, optional\n          For which remote (based on uuid) to report description for\n\n        Returns\n        -------\n        str or None\n          None returned if not found\n        \"\"\"\n        info = self.repo_info(fast=True)\n        match = \\\n            (lambda x: x['here']) \\\n            if uuid is None \\\n            else (lambda x: x['uuid'] == uuid)\n\n        matches = list(set(chain.from_iterable(\n            [\n                [r['description'] for r in remotes if match(r)]\n                for k, remotes in iteritems(info)\n                if k.endswith(' repositories')\n            ]\n        )))\n\n        if len(matches) == 1:\n            # single hit as it should\n            return matches[0]\n        elif len(matches) == 2:\n            lgr.warning(\n                \"Found multiple hits while searching. Returning first among: %s\",\n                str(matches)\n            )\n            return matches[0]\n        else:\n            return None\n\n    def set_remote_url(self, name, url, push=False):\n        \"\"\"Set the URL a remote is pointing to\n\n        Sets the URL of the remote `name`. Requires the remote to already exist.\n\n        Parameters\n        ----------\n        name: str\n          name of the remote\n        url: str\n        push: bool\n          if True, set the push URL, otherwise the fetch URL;\n          if True, additionally set annexurl to `url`, to make sure annex uses\n          it to talk to the remote, since access via fetch URL might be\n          restricted.\n        \"\"\"\n\n        if push:\n            # if we are to set a push url, also set 'annexUrl' for this remote,\n            # in order to make git-annex use it, when talking to the remote.\n            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)\n            var = 'remote.{0}.{1}'.format(name, 'annexurl')\n            self.config.set(var, url, where='local', reload=True)\n        super(AnnexRepo, self).set_remote_url(name, url, push)\n\n    def get_metadata(self, files, timestamps=False):\n        \"\"\"Query git-annex file metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be queried.\n        timestamps: bool, optional\n          If True, the output contains a '<metadatakey>-lastchanged'\n          key for every metadata item, reflecting the modification\n          time, as well as a 'lastchanged' key with the most recent\n          modification time of any metadata item.\n\n        Returns\n        -------\n        generator\n          One tuple per file (could be more items than input arguments\n          when directories are given). First tuple item is the filename,\n          second item is a dictionary with metadata key/value pairs. Note that annex\n          metadata tags are stored under the key 'tag', which is a\n          regular metadata item that can be manipulated like any other.\n        \"\"\"\n        if not files:\n            return\n        files = assure_list(files)\n        opts = ['--json']\n        for res in self._run_annex_command_json(\n                'metadata', opts=opts, files=files):\n            yield (\n                res['file'],\n                res['fields'] if timestamps else \\\n                {k: v for k, v in res['fields'].items()\n                 if not k.endswith('lastchanged')})\n\n    def set_metadata(\n            self, files, reset=None, add=None, init=None,\n            remove=None, purge=None, recursive=False):\n        \"\"\"Manipulate git-annex file-metadata\n\n        Parameters\n        ----------\n        files : str or list(str)\n          One or more paths for which metadata is to be manipulated.\n          The changes applied to each file item are uniform. However,\n          the result may not be uniform across files, depending on the\n          actual operation.\n        reset : dict, optional\n          Metadata items matching keys in the given dict are (re)set\n          to the respective values.\n        add : dict, optional\n          The values of matching keys in the given dict appended to\n          any possibly existing values. The metadata keys need not\n          necessarily exist before.\n        init : dict, optional\n          Metadata items for the keys in the given dict are set\n          to the respective values, if the key is not yet present\n          in a file's metadata.\n        remove : dict, optional\n          Values in the given dict are removed from the metadata items\n          matching the respective key, if they exist in a file's metadata.\n          Non-existing values, or keys do not lead to failure.\n        purge : list, optional\n          Any metadata item with a key matching an entry in the given\n          list is removed from the metadata.\n        recursive : bool, optional\n          If False, fail (with CommandError) when directory paths\n          are given as `files`.\n\n        Returns\n        -------\n        generator\n          JSON obj per modified file\n        \"\"\"\n\n        def _genspec(expr, d):\n            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]\n\n        args = []\n        spec = []\n        for expr, d in (('{}={}', reset),\n                        ('{}+={}', add),\n                        ('{}?={}', init),\n                        ('{}-={}', remove)):\n            if d:\n                spec.extend(_genspec(expr, d))\n        # prefix all with '-s' and extend arg list\n        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)\n        if purge:\n            # and all '-r' args\n            args.extend(j for i in zip(['-r'] * len(purge), purge)\n                        for j in i)\n        if not args:\n            return\n\n        if recursive:\n            args.append('--force')\n\n        # Make sure that batch add/addurl operations are closed so that we can\n        # operate on files that were just added.\n        self.precommit()\n\n        for jsn in self._run_annex_command_json(\n                'metadata',\n                args,\n                files=files):\n            yield jsn\n\n    @staticmethod\n    def _is_annex_work_tree_message(out):\n        return re.match(\n            r'.*This operation must be run in a work tree.*'\n            r'git status.*failed in submodule',\n            out,\n            re.MULTILINE | re.DOTALL | re.IGNORECASE)\n\n# TODO: Why was this commented out?\n# @auto_repr\nclass BatchedAnnexes(dict):\n    \"\"\"Class to contain the registry of active batch'ed instances of annex for\n    a repository\n    \"\"\"\n    def __init__(self, batch_size=0, git_options=None):\n        self.batch_size = batch_size\n        self.git_options = git_options or []\n        super(BatchedAnnexes, self).__init__()\n\n    def get(self, codename, annex_cmd=None, **kwargs):\n        if annex_cmd is None:\n            annex_cmd = codename\n\n        git_options = self.git_options + kwargs.pop('git_options', [])\n        if self.batch_size:\n            git_options += ['-c', 'annex.queuesize=%d' % self.batch_size]\n\n        # START RF/BF: extend codename to respect different options the process\n        # is running with\n        # TODO: Eventually there should be more RF'ing, since the actually used\n        # codenames are partially reflecting this already. Any options used\n        # therein should go away, since they are now automatically included.\n        options = kwargs.copy()\n        options['git_options'] = git_options\n        options['annex_cmd'] = annex_cmd\n        for key in options:\n            codename += ':{0}:{1}'.format(key, options[key])\n        # END RF/BF\n\n        if codename not in self:\n            # Create a new git-annex process we will keep around\n            self[codename] = BatchedAnnex(annex_cmd,\n                                          git_options=git_options,\n                                          **kwargs)\n        return self[codename]\n\n    def clear(self):\n        \"\"\"Override just to make sure we don't rely on __del__ to close all\n        the pipes\"\"\"\n        self.close()\n        super(BatchedAnnexes, self).clear()\n\n    def close(self):\n        \"\"\"Close communication to all the batched annexes\n\n        It does not remove them from the dictionary though\n        \"\"\"\n        for p in self.values():\n            p.close()\n\n    def __del__(self):\n        self.close()\n\n\ndef readline_rstripped(stdout):\n    #return iter(stdout.readline, b'').next().rstrip()\n    return stdout.readline().rstrip()\n\n\ndef readlines_until_ok_or_failed(stdout, maxlines=100):\n    \"\"\"Read stdout until line ends with ok or failed\"\"\"\n    out = ''\n    i = 0\n    lgr.log(3, \"Trying to receive from %s\" % stdout)\n    while not stdout.closed:\n        i += 1\n        if maxlines > 0 and i > maxlines:\n            raise IOError(\"Expected no more than %d lines. So far received: %r\" % (maxlines, out))\n        lgr.log(2, \"Expecting a line\")\n        line = stdout.readline()\n        lgr.log(2, \"Received line %r\" % line)\n        out += line\n        if re.match(r'^.*\\b(failed|ok)$', line.rstrip()):\n            break\n    return out.rstrip()\n\n\ndef readline_json(stdout):\n    toload = stdout.readline().strip()\n    return json_loads(toload) if toload else {}\n\n\n@auto_repr\nclass BatchedAnnex(object):\n    \"\"\"Container for an annex process which would allow for persistent communication\n    \"\"\"\n\n    def __init__(self, annex_cmd, git_options=None, annex_options=None, path=None,\n                 json=False,\n                 output_proc=None):\n        if not isinstance(annex_cmd, list):\n            annex_cmd = [annex_cmd]\n        self.annex_cmd = annex_cmd\n        self.git_options = git_options if git_options else []\n        annex_options = annex_options if annex_options else []\n        self.annex_options = annex_options + (['--json'] if json else [])\n        self.path = path\n        if output_proc is None:\n            output_proc = readline_json if json else readline_rstripped\n        self.output_proc = output_proc\n        self._process = None\n        self._stderr_out = None\n        self._stderr_out_fname = None\n\n    def _initialize(self):\n        # TODO -- should get all those options about --debug and --backend which are used/composed\n        # in AnnexRepo class\n        lgr.debug(\"Initiating a new process for %s\" % repr(self))\n        cmd = ['git'] + self.git_options + \\\n              ['annex'] + self.annex_cmd + self.annex_options + ['--batch']  # , '--debug']\n        lgr.log(5, \"Command: %s\" % cmd)\n        # TODO: look into _run_annex_command  to support default options such as --debug\n        #\n        # according to the internet wisdom there is no easy way with subprocess\n        # while avoid deadlocks etc.  We would need to start a thread/subprocess\n        # to timeout etc\n        # kwargs = dict(bufsize=1, universal_newlines=True) if PY3 else {}\n        self._stderr_out, self._stderr_out_fname = tempfile.mkstemp()\n        self._process = Popen(\n            cmd, stdin=PIPE, stdout=PIPE,  stderr=self._stderr_out,\n            env=GitRunner.get_git_environ_adjusted(),\n            cwd=self.path,\n            bufsize=1,\n            universal_newlines=True  # **kwargs\n        )\n\n    def _check_process(self, restart=False):\n        \"\"\"Check if the process was terminated and restart if restart\n\n        Returns\n        -------\n        bool\n          True if process was alive.\n        str\n          stderr if any recorded if was terminated\n        \"\"\"\n        process = self._process\n        ret = True\n        ret_stderr = None\n        if process and process.poll():\n            lgr.warning(\"Process %s was terminated with returncode %s\" % (process, process.returncode))\n            ret_stderr = self.close(return_stderr=True)\n            ret = False\n        if self._process is None and restart:\n            lgr.warning(\"Restarting the process due to previous failure\")\n            self._initialize()\n        return ret, ret_stderr\n\n    def __call__(self, cmds):\n        \"\"\"\n\n        Parameters\n        ----------\n        cmds : str or tuple or list of (str or tuple)\n\n        Returns\n        -------\n        str or list\n          Output received from annex.  list in case if cmds was a list\n        \"\"\"\n        # TODO: add checks -- may be process died off and needs to be reinitiated\n        if not self._process:\n            self._initialize()\n\n        input_multiple = isinstance(cmds, list)\n        if not input_multiple:\n            cmds = [cmds]\n\n        output = []\n\n        for entry in cmds:\n            if not isinstance(entry, string_types):\n                entry = ' '.join(entry)\n            entry = entry + '\\n'\n            lgr.log(5, \"Sending %r to batched annex %s\" % (entry, self))\n            # apparently communicate is just a one time show\n            # stdout, stderr = self._process.communicate(entry)\n            # according to the internet wisdom there is no easy way with subprocess\n            self._check_process(restart=True)\n            process = self._process  # _check_process might have restarted it\n            process.stdin.write(assure_bytes(entry) if PY2 else entry)\n            process.stdin.flush()\n            lgr.log(5, \"Done sending.\")\n            still_alive, stderr = self._check_process(restart=False)\n            # TODO: we might want to handle still_alive, e.g. to allow for\n            #       a number of restarts/resends, but it should be per command\n            #       since for some we cannot just resend the same query. But if\n            #       it is just a \"get\"er - we could resend it few times\n            # We are expecting a single line output\n            # TODO: timeouts etc\n            stdout = assure_unicode(self.output_proc(process.stdout)) \\\n                if not process.stdout.closed else None\n            if stderr:\n                lgr.warning(\"Received output in stderr: %r\", stderr)\n            lgr.log(5, \"Received output: %r\" % stdout)\n            output.append(stdout)\n\n        return output if input_multiple else output[0]\n\n    def __del__(self):\n        self.close()\n\n    def close(self, return_stderr=False):\n        \"\"\"Close communication and wait for process to terminate\n\n        Returns\n        -------\n        str\n          stderr output if return_stderr and stderr file was there.\n          None otherwise\n        \"\"\"\n        ret = None\n        if self._stderr_out:\n            # close possibly still open fd\n            os.fdopen(self._stderr_out).close()\n            self._stderr_out = None\n        if self._process:\n            process = self._process\n            lgr.debug(\n                \"Closing stdin of %s and waiting process to finish\", process)\n            process.stdin.close()\n            process.stdout.close()\n            process.wait()\n            self._process = None\n            lgr.debug(\"Process %s has finished\", process)\n        if self._stderr_out_fname and os.path.exists(self._stderr_out_fname):\n            if return_stderr:\n                with open(self._stderr_out_fname, 'r') as f:\n                    ret = f.read()\n            # remove the file where we kept dumping stderr\n            unlink(self._stderr_out_fname)\n            self._stderr_out_fname = None\n        return ret\n\n\ndef _get_size_from_perc_complete(count, perc):\n    \"\"\"A helper to get full size if know % and corresponding size\"\"\"\n    portion = (float(perc) / 100.)\n    return int(math.ceil(int(count) / portion)) \\\n        if portion else 0\n\n\nclass ProcessAnnexProgressIndicators(object):\n    \"\"\"'Filter' for annex --json output to react to progress indicators\n\n    Instance of this beast should be passed into log_stdout option\n    for git-annex commands runner\n    \"\"\"\n\n    def __init__(self, expected=None):\n        \"\"\"\n\n        Parameters\n        ----------\n        expected: dict, optional\n           key -> size, expected entries (e.g. downloads)\n        \"\"\"\n        # looking forward for multiple downloads at the same time\n        self.pbars = {}\n        self.total_pbar = None\n        self.expected = expected\n        self.only_one_expected = expected and len(self.expected) == 1\n        self._failed = 0\n        self._succeeded = 0\n        self.start()\n\n    def start(self):\n        if self.expected:\n            from datalad.ui import ui\n            total = sum(filter(bool, self.expected.values()))\n            if len(self.expected) > 1:\n                self.total_pbar = ui.get_progressbar(\n                    label=\"Total\", total=total)\n                self.total_pbar.start()\n\n    def __getitem__(self, id_):\n        if self.pbars == 1:\n            # regardless what it is, it is the one!\n            # will happen e.g. in case of add_url_to_file since filename\n            # is not even reported in the status and final msg just\n            # reports the key which is not known before\n            return self.pbars.items()[0]\n        return self.pbars[id_]\n\n    def _update_pbar(self, pbar, new_value):\n        \"\"\"Updates pbar while also updating possibly total pbar\"\"\"\n        old_value = getattr(pbar, '_old_value', 0)\n        # due to http://git-annex.branchable.com/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/?updated\n        # we will just skip the first update to avoid possible incorrect\n        # reporting\n        if not getattr(pbar, '_first_skipped', False):\n            setattr(pbar, '_first_skipped', True)\n            lgr.log(1, \"Skipped first update of pbar %s\", pbar)\n            return\n        setattr(pbar, '_old_value', new_value)\n        diff = new_value - old_value\n        if diff < 0:\n            # so above didn't help!\n            # use warnings not lgr.warn since we apparently swallow stuff\n            # upstairs!  Also it would take care about issuing it only once\n            lgr.debug(\n                \"Got negative diff for progressbar. old_value=%r, new_value=%r\",\n                old_value, new_value)\n            return\n        if self.total_pbar:\n            self.total_pbar.update(diff, increment=True)\n        pbar.update(new_value)\n\n    def _log_info(self, msg):\n        \"\"\"Helper to log a message, so we need to clear up the pbars first\"\"\"\n        if self.total_pbar:\n            self.total_pbar.clear()\n        for pbar in self.pbars.values():\n            pbar.clear()\n        lgr.info(msg)\n\n    def __call__(self, line):\n        try:\n            j = json.loads(line)\n        except:\n            # if we fail to parse, just return this precious thing for\n            # possibly further processing\n            return line\n        # Process some messages which remotes etc might push to us\n        if list(j) == ['info']:\n            # Just INFO was received without anything else -- we log it at INFO\n            info = j['info']\n            if info.startswith('PROGRESS-JSON: '):\n                j_ = json_loads(info[len('PROGRESS-JSON: '):])\n                if ('command' in j_ and 'key' in j_) or 'byte-progress' in j_:\n                    j = j_\n                else:\n                    self._log_info(info)\n            else:\n                self._log_info(info)\n                return\n\n        target_size = None\n        if 'command' in j and ('key' in j or 'file' in j):\n            # might be the finish line message\n            action_item = j.get('key') or j.get('file')\n            j_action_id = (j['command'], action_item)\n            pbar = self.pbars.pop(j_action_id, None)\n            if pbar is None and len(self.pbars) == 1 and self.only_one_expected:\n                # it is the only one left - take it!\n                pbar = self.pbars.popitem()[1]\n            if j.get('success') in {True, 'true'}:\n                self._succeeded += 1\n                if pbar:\n                    self._update_pbar(pbar, pbar.total)\n                elif self.total_pbar:\n                    # we didn't have a pbar for this download, so total should\n                    # get it all at once\n                    try:\n                        target_size = self.expected[action_item]\n                    except:\n                        target_size = None\n                    if not target_size and 'key' in j:\n                        target_size = AnnexRepo.get_size_from_key(j['key'])\n                    self.total_pbar.update(target_size, increment=True)\n            else:\n                lgr.log(5, \"Message with failed status: %s\" % str(j))\n                self._failed += 1\n\n            if self.total_pbar:\n                failed_str = (\n                    \", \" + ansi_colors.color_word(\"%d failed\" % self._failed,\n                                                  ansi_colors.RED)) \\\n                    if self._failed else ''\n\n                self.total_pbar.set_desc(\n                    \"Total (%d ok%s out of %d)\" % (\n                        self._succeeded,\n                        failed_str,\n                        len(self.expected)\n                        if self.expected\n                        else self._succeeded + self._failed))\n                # seems to be of no effect to force it repaint\n                self.total_pbar.refresh()\n\n            if pbar:\n                pbar.finish()\n\n        if 'byte-progress' not in j:\n            # some other thing than progress\n            return line\n\n        # so we have a progress indicator, let's deal with it\n        action = j['action']\n        action_item = action.get('key') or action.get('file')\n        action_id = (action['command'], action_item)\n        if action_id not in self.pbars:\n            # New download!\n            from datalad.ui import ui\n            from datalad.ui import utils as ui_utils\n            # TODO: whenever target size gets reported -- used it!\n            # http://git-annex.branchable.com/todo/interface_to_the___34__progress__34___of_annex_operations/#comment-6bbc26aae9867603863050ddcb09a9a0\n            # for now deduce from key or approx from '%'\n            # TODO: unittest etc to check when we have a relaxed\n            # URL without any size known in advance\n            if not target_size:\n                target_size = \\\n                    AnnexRepo.get_size_from_key(action.get('key')) or \\\n                    _get_size_from_perc_complete(\n                        j['byte-progress'],\n                        j.get('percent-progress', '').rstrip('%')\n                    ) or \\\n                    0\n            w = ui_utils.get_console_width()\n\n            if not action_item and self.only_one_expected:\n                # must be the one!\n                action_item = list(self.expected)[0]\n\n            title = str(action.get('file') or action_item)\n\n            pbar_right = 50\n            title_len = w - pbar_right - 4  # (4 for reserve)\n            if len(title) > title_len:\n                half = title_len//2 - 2\n                title = '%s .. %s' % (title[:half], title[-half:])\n            pbar = self.pbars[action_id] = ui.get_progressbar(\n                label=title, total=target_size)\n            pbar.start()\n\n        lgr.log(1, \"Updating pbar for action_id=%s. annex: %s.\\n\",\n                action_id, j)\n        self._update_pbar(\n            self[action_id],\n            int(j.get('byte-progress'))\n        )\n\n    def finish(self):\n        if self.total_pbar:\n            self.total_pbar.finish()\n            self.total_pbar = None\n        if self.pbars:\n            lgr.warning(\"Still have %d active progress bars when stopping\",\n                        len(self.pbars))\n        for pbar in self.pbars.values():\n            pbar.finish()\n        self.pbars = {}\n        self._failed = 0\n        self._succeeded = 0\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** `_get_size_from_perc_complete` crashes when percentage is missing

**Description:**
When the percentage value is not provided, the `_get_size_from_perc_complete` function fails instead of handling the missing input gracefully.

**Example Code:**
```python
result = _get_size_from_perc_complete(1, '')
```

**Expected Behavior:**
The function should return `0` when the percentage is not provided.

**Actual Behavior:**
A `ValueError` is raised because it attempts to convert an empty string to a float.
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/support/annexrepo.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Interface to git-annex by Joey Hess.

For further information on git-annex see https://git-annex.branchable.com/.

"""

import json
import logging
import math
import os
import re
import shlex
import tempfile
import time

from itertools import chain
from os import linesep
from os.path import curdir
from os.path import join as opj
from os.path import exists
from os.path import islink
from os.path import realpath
from os.path import lexists
from os.path import isdir
from os.path import isabs
from os.path import relpath
from os.path import normpath
from subprocess import Popen, PIPE
from multiprocessing import cpu_count
from weakref import WeakValueDictionary

from six import string_types, PY2
from six import iteritems
from six.moves import filter
from git import InvalidGitRepositoryError

from datalad import ssh_manager
from datalad.dochelpers import exc_str
from datalad.dochelpers import borrowdoc
from datalad.dochelpers import borrowkwargs
from datalad.utils import linux_distribution_name
from datalad.utils import nothing_cm
from datalad.utils import auto_repr
from datalad.utils import on_windows
from datalad.utils import swallow_logs
from datalad.utils import assure_list
from datalad.utils import _path_
from datalad.utils import generate_chunks
from datalad.utils import CMD_MAX_ARG
from datalad.utils import assure_unicode, assure_bytes
from datalad.utils import make_tempfile
from datalad.utils import unlink
from datalad.support.json_py import loads as json_loads
from datalad.cmd import GitRunner

# imports from same module:
from .repo import RepoInterface
from .gitrepo import GitRepo
from .gitrepo import NoSuchPathError
from .gitrepo import _normalize_path
from .gitrepo import normalize_path
from .gitrepo import normalize_paths
from .gitrepo import GitCommandError
from .gitrepo import to_options
from . import ansi_colors
from .external_versions import external_versions
from .exceptions import CommandNotAvailableError
from .exceptions import CommandError
from .exceptions import FileNotInAnnexError
from .exceptions import FileInGitError
from .exceptions import FileNotInRepositoryError
from .exceptions import AnnexBatchCommandError
from .exceptions import InsufficientArgumentsError
from .exceptions import OutOfSpaceError
from .exceptions import RemoteNotAvailableError
from .exceptions import OutdatedExternalDependency
from .exceptions import MissingExternalDependency
from .exceptions import IncompleteResultsError
from .exceptions import AccessDeniedError
from .exceptions import AccessFailedError

lgr = logging.getLogger('datalad.annex')

# Limit to # of CPUs and up to 8, but at least 3 to start with
N_AUTO_JOBS = min(8, max(3, cpu_count()))


class AnnexRepo(GitRepo, RepoInterface):
    """Representation of an git-annex repository.

    Paths given to any of the class methods will be interpreted as relative
    to PWD, in case this is currently beneath AnnexRepo's base dir
    (`self.path`). If PWD is outside of the repository, relative paths
    will be interpreted as relative to `self.path`. Absolute paths will be
    accepted either way.
    """

    # Begin Flyweight:
    _unique_instances = WeakValueDictionary()
    # End Flyweight:

    # Web remote has a hard-coded UUID we might (ab)use
    WEB_UUID = "00000000-0000-0000-0000-000000000001"

    # To be assigned and checked to be good enough upon first call to AnnexRepo
    # 6.20160923 -- --json-progress for get
    # 6.20161210 -- annex add  to add also changes (not only new files) to git
    # 6.20170220 -- annex status provides --ignore-submodules
    # 6.20180416 -- annex handles unicode filenames more uniformly
    # 6.20180913 -- annex fixes all known to us issues for v6
    GIT_ANNEX_MIN_VERSION = '6.20180913'
    git_annex_version = None

    # Class wide setting to allow insecure URLs. Used during testing, since
    # git annex 6.20180626 those will by default be not allowed for security
    # reasons
    _ALLOW_LOCAL_URLS = False

    def __init__(self, path, url=None, runner=None,
                 direct=None, backend=None, always_commit=True, create=True,
                 init=False, batch_size=None, version=None, description=None,
                 git_opts=None, annex_opts=None, annex_init_opts=None,
                 repo=None, fake_dates=False):
        """Creates representation of git-annex repository at `path`.

        AnnexRepo is initialized by giving a path to the annex.
        If no annex exists at that location, a new one is created.
        Optionally give url to clone from.

        Parameters
        ----------
        path: str
          Path to git-annex repository. In case it's not an absolute path, it's
          relative to PWD
        url: str, optional
          url to the to-be-cloned repository. Requires valid git url
          according to
          http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .
        runner: Runner, optional
          Provide a Runner in case AnnexRepo shall not create it's own.
          This is especially needed in case of desired dry runs.
        direct: bool, optional
          If True, force git-annex to use direct mode
        backend: str, optional
          Set default backend used by this annex. This does NOT affect files,
          that are already annexed nor will it automatically migrate files,
          hat are 'getted' afterwards.
        create: bool, optional
          Create and initialize an annex repository at path, in case
          there is none. If set to False, and this repository is not an annex
          repository (initialized or not), an exception is raised.
        init: bool, optional
          Initialize git-annex repository (run "git annex init") if path is an
          annex repository which just was not yet initialized by annex (e.g. a
          fresh git clone). Note that if `create=True`, then initialization
          would happen
        batch_size: int, optional
          If specified and >0, instructs annex to batch this many commands before
          annex adds acts on git repository (e.g. adds them them to index for addurl).
        version: int, optional
          If given, pass as --version to `git annex init`
        description: str, optional
          Short description that humans can use to identify the
          repository/location, e.g. "Precious data on my laptop"
        """
        if self.git_annex_version is None:
            self._check_git_annex_version()

        # initialize
        self._uuid = None
        self._annex_common_options = []
        # Workaround for per-call config issue with git 2.11.0
        self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = False
        self.GIT_DIRECT_MODE_PROXY = False

        if annex_opts or annex_init_opts:
            lgr.warning("TODO: options passed to git-annex and/or "
                        "git-annex-init are currently ignored.\n"
                        "options received:\n"
                        "git-annex: %s\ngit-annex-init: %s" %
                        (annex_opts, annex_init_opts))

        fix_it = False
        try:
            super(AnnexRepo, self).__init__(path, url, runner=runner,
                                            create=create, repo=repo,
                                            git_opts=git_opts,
                                            fake_dates=fake_dates)
        except GitCommandError as e:
            if create and "Clone succeeded, but checkout failed." in str(e):
                lgr.warning("Experienced issues while cloning. "
                            "Trying to fix it, using git-annex-fsck.")
                fix_it = True
            else:
                raise e

        # Below while setting for direct mode workaround, we change
        # _GIT_COMMON_OPTIONS.
        # But we should not pass workarounds such as --worktree=.
        # -c core.bare=False to git annex commands, so for their
        # invocation we will keep and use pristine version of the
        # common options
        self._ANNEX_GIT_COMMON_OPTIONS = self._GIT_COMMON_OPTIONS[:]

        # check for possible SSH URLs of the remotes in order to set up
        # shared connections:
        for r in self.get_remotes():
            for url in [self.get_remote_url(r),
                        self.get_remote_url(r, push=True)]:
                if url is not None:
                    self._set_shared_connection(r, url)

        self.always_commit = always_commit

        if version is None:
            version = self.config.get("datalad.repo.version", None)
            # we might get an empty string here
            # TODO: if we use obtain() instead, we get an error complaining
            # '' cannot be converted to int (via Constraint as defined for
            # "datalad.repo.version" in common_cfg
            # => Allow conversion to result in None?
            if not version:
                version = None

        if fix_it:
            self._init(version=version, description=description)
            self.fsck()

        # Check whether an annex already exists at destination
        # XXX this doesn't work for a submodule!
        if not AnnexRepo.is_valid_repo(self.path):
            # so either it is not annex at all or just was not yet initialized
            if self.is_with_annex():
                # it is an annex repository which was not initialized yet
                if create or init:
                    lgr.debug('Annex repository was not yet initialized at %s.'
                              ' Initializing ...' % self.path)
                    self._init(version=version, description=description)
            elif create:
                lgr.debug('Initializing annex repository at %s...' % self.path)
                self._init(version=version, description=description)
            else:
                raise RuntimeError("No annex found at %s." % self.path)

        self._direct_mode = None  # we don't know yet

        # If we are in direct mode already, we need to make
        # this instance aware of that. This especially means, that we need to
        # adapt self._GIT_COMMON_OPTIONS by calling set_direct_mode().
        # Could happen in case we didn't specify anything, but annex forced
        # direct mode due to FS or an already existing repo was in direct mode,
        if self._is_direct_mode_from_config():
            self.set_direct_mode()

        # - only force direct mode; don't force indirect mode
        # - parameter `direct` has priority over config
        if direct is None:
            direct = (create or init) and \
                     self.config.getbool("datalad", "repo.direct", default=False)
        self._direct_mode = None  # we don't know yet
        if direct and not self.is_direct_mode():
            # direct mode is available below version 6 repos only.
            # Note: If 'annex.version' is missing in .git/config for some
            # reason, we need to try to set direct mode:
            repo_version = self.config.getint("annex", "version")
            if (repo_version is None) or (repo_version < 6):
                lgr.debug("Switching to direct mode (%s)." % self)
                self.set_direct_mode()
            else:
                # TODO: This may change to either not being a warning and/or
                # to use 'git annex unlock' instead.
                lgr.warning("direct mode not available for %s. Ignored." % self)

        self._batched = BatchedAnnexes(
            batch_size=batch_size, git_options=self._ANNEX_GIT_COMMON_OPTIONS)

        # set default backend for future annex commands:
        # TODO: Should the backend option of __init__() also migrate
        # the annex, in case there are annexed files already?
        if backend:
            self.set_default_backend(backend, persistent=True)

        if self._ALLOW_LOCAL_URLS:
            self._allow_local_urls()

    def _allow_local_urls(self):
        """Allow URL schemes and addresses which potentially could be harmful.

        For now it is internal method used within tests only
        """
        # from annex 6.20180626 file:/// and http://localhost access isn't
        # allowed by default
        self.config.add(
            'annex.security.allowed-url-schemes', 'http https file',
            'local')
        self.config.add(
            'annex.security.allowed-http-addresses', 'all',
            'local')

    def set_default_backend(self, backend, persistent=True, commit=True):
        """Set default backend

        Parameters
        ----------
        backend : str
        persistent : bool, optional
          If persistent, would add/commit to .gitattributes. If not -- would
          set within .git/config
        """
        # TODO: 'annex.backends' actually is a space separated list.
        # Figure out, whether we want to allow for a list here or what to
        # do, if there is sth in that setting already
        if persistent:
            # could be set in .gitattributes or $GIT_DIR/info/attributes
            if 'annex.backend' in self.get_gitattributes('.')['.']:
                lgr.debug(
                    "Not (re)setting backend since seems already set in git attributes"
                )
            else:
                lgr.debug("Setting annex backend to %s (persistently)", backend)
                self.config.set('annex.backends', backend, where='local')
                git_attributes_file = _path_(self.path, '.gitattributes')
                git_attributes = ''
                if exists(git_attributes_file):
                    with open(git_attributes_file) as f:
                        git_attributes = f.read()
                with open(git_attributes_file, 'a') as f:
                    if git_attributes and not git_attributes.endswith(os.linesep):
                        f.write(os.linesep)
                    f.write('* annex.backend=%s%s' % (backend, os.linesep))
                self.add(git_attributes_file, git=True)
                if commit:
                    self.commit(
                        "Set default backend for all files to be %s" % backend,
                        _datalad_msg=True,
                        files=[git_attributes_file]
                    )
        else:
            lgr.debug("Setting annex backend to %s (in .git/config)", backend)
            self.config.set('annex.backends', backend, where='local')

    def __del__(self):
        try:
            if hasattr(self, '_batched') and self._batched is not None:
                self._batched.close()
        except TypeError as e:
            # Workaround:
            # most likely something wasn't accessible anymore; doesn't really
            # matter since we wanted to delete it anyway.
            #
            # Nevertheless, in some cases might be an issue and it is a strange
            # thing to happen, since we check for things being None herein as
            # well as in super class __del__;
            # At least log it:
            lgr.debug(exc_str(e))
        try:
            super(AnnexRepo, self).__del__()
        except TypeError as e:
            # see above
            lgr.debug(exc_str(e))

    def _set_shared_connection(self, remote_name, url):
        """Make sure a remote with SSH URL uses shared connections.

        Set ssh options for annex on a per call basis, using
        '-c remote.<name>.annex-ssh-options'.

        Note
        ----
        There's currently no solution for using these connections, if the SSH
        URL is just connected to a file instead of a remote
        (`annex addurl` for example).

        Parameters
        ----------
        remote_name: str
        url: str
        """
        from datalad.support.network import is_ssh
        # Note:
        #
        # before any possible invocation of git-annex
        # Temporary approach to ssh connection sharing:
        # Register every ssh remote with the corresponding control master.
        # Issues:
        # - currently overwrites existing ssh config of the remote
        # - request SSHConnection instance and write config even if no
        #   connection needed (but: connection is not actually created/opened)
        # - no solution for a ssh url of a file (annex addurl)

        if is_ssh(url):
            c = ssh_manager.get_connection(url)
            ssh_cfg_var = "remote.{0}.annex-ssh-options".format(remote_name)
            # options to add:
            # Note: must use -S to overload -S provided by annex itself
            # if we provide -o ControlPath=... it is not in effect
            # Note: ctrl_path must not contain spaces, since it seems to be
            # impossible to anyhow guard them here
            # http://git-annex.branchable.com/bugs/cannot___40__or_how__63____41___to_pass_socket_path_with_a_space_in_its_path_via_annex-ssh-options/
            cfg_string = "-o ControlMaster=auto -S %s" % c.ctrl_path
            # read user-defined options from .git/config:
            cfg_string_old = self.config.get(ssh_cfg_var, None)
            self._annex_common_options += \
                ['-c', 'remote.{0}.annex-ssh-options={1}{2}'
                       ''.format(remote_name,
                                 (cfg_string_old + " ") if cfg_string_old else "",
                                 cfg_string
                                 )]

    def is_managed_branch(self, branch=None):
        """Whether `branch` is managed by git-annex.

        ATM this returns true in direct mode (branch 'annex/direct/my_branch')
        and if on an adjusted branch (annex v6 repository:
        either 'adjusted/my_branch(unlocked)' or 'adjusted/my_branch(fixed)'

        Note: The term 'managed branch' is used to make clear it's meant to be
        more general than the v6 'adjusted branch'.

        Parameters
        ----------
        branch: str
          name of the branch; default: active branch

        Returns
        -------
        bool
          True if on a managed branch, False otherwise
        """

        if branch is None:
            branch = self.get_active_branch()
        # Note: `branch` might still be None, due to detached HEAD
        # (or no checkout at all)
        if branch and \
            (branch.startswith('annex/direct/') or
             branch.startswith('adjusted/')):
            return True
        return False

    def get_corresponding_branch(self, branch=None):
        """In case of a managed branch, get the corresponding one.

        If `branch` is not a managed branch, return that branch without any
        changes.

        Note: Since default for `branch` is the active branch,
        `get_corresponding_branch()` is equivalent to `get_active_branch()` if
        the active branch is not a managed branch.

        Parameters
        ----------
        branch: str
          name of the branch; defaults to active branch

        Returns
        -------
        str
          name of the corresponding branch if there is any, name of the queried
          branch otherwise.
        """

        if branch is None:
            branch = self.get_active_branch()

        if self.is_managed_branch(branch):
            if branch.startswith('annex/direct/'):
                cor_branch = branch[13:]
            elif branch.startswith('adjusted/'):
                if branch.endswith('(unlocked)'):
                    cor_branch = branch[9:-10]
                elif branch.endswith('(fixed)'):
                    cor_branch = branch[9:-7]
                else:
                    cor_branch = branch[9:]
                    lgr.warning("Unexpected naming of adjusted branch '{}'.{}"
                                "Assuming '{}' to be the corresponding branch."
                                "".format(branch, linesep, cor_branch))
            else:
                raise NotImplementedError(
                    "Detection of annex-managed branch '{}' follows a pattern "
                    "not implemented herein.".format(branch))
            return cor_branch

        else:
            return branch

    def get_tracking_branch(self, branch=None, corresponding=True):
        """Get the tracking branch for `branch` if there is any.

        By default returns the tracking branch of the corresponding branch if
        `branch` is a managed branch.

        Parameters
        ----------
        branch: str
          local branch to look up. If none is given, active branch is used.
        corresponding: bool
          If True actually look up the corresponding branch of `branch` (also if
          `branch` isn't explicitly given)

        Returns
        -------
        tuple
            (remote or None, refspec or None) of the tracking branch
        """

        if branch is None:
            branch = self.get_active_branch()

        return super(AnnexRepo, self).get_tracking_branch(
                        branch=self.get_corresponding_branch(branch)
                        if corresponding else branch)

    def _submodules_dirty_direct_mode(self,
            untracked=True, deleted=True, modified=True, added=True,
            type_changed=True, path=None):
        """Get modified submodules

        Workaround for http://git-annex.branchable.com/bugs/git_annex_status_fails_with_submodule_in_direct_mode/

        This is using git-annex-status with --ignore-submodules to not let
        git-status try to recurse into annex submodules without a working tree.
        Therefore we need to do the recursion on our own.

        Note, that added submodules will just be reported dirty. It's at very
        least difficult to distinguish whether a submodule in direct mode was
        just added or modified. ATM not worth the effort, I think.
        This is leads to a bit inconsistent reportings by AnnexRepo.status()
        whenever it needs to call this subroutine and there are added submodules.

        Intended to be used by AnnexRepo.status() internally.
        """

        # Note: We do a lazy recursion. The only thing we need to know is
        # whether or not a submodule is to be reported dirty. Once we already
        # know it is, there's no need to go any deeper in the hierarchy.
        # Apart from better performance, this also allows us to inspect each
        # submodule separately, and therefore be able to deal with mixed
        # hierarchies of git and annex submodules!

        modified_subs = []
        for sm in self.get_submodules():
            sm_dirty = False

            # First check for changes committed in the submodule, using
            # git submodule summary -- path,
            # since this can't be detected from within the submodule.
            if self.is_submodule_modified(sm.name):
                sm_dirty = True

            # check state of annex submodules, that might be in direct mode
            elif AnnexRepo.is_valid_repo(opj(self.path, sm.path),
                                         allow_noninitialized=False):

                sm_repo = AnnexRepo(opj(self.path, sm.path),
                                    create=False, init=False)

                sm_status = sm_repo.get_status(untracked=untracked, deleted=deleted,
                                               modified=modified, added=added,
                                               type_changed=type_changed,
                                               submodules=False, path=path)
                if any([bool(sm_status[i]) for i in sm_status]):
                    sm_dirty = True

            # check state of submodule, that is a plain git or not an
            # initialized annex, which we can safely treat as a plain git, too.
            elif GitRepo.is_valid_repo(opj(self.path, sm.path)):
                sm_repo = GitRepo(opj(self.path, sm.path))

                # TODO: Clarify issue: GitRepo.is_dirty() doesn't fit our parameters
                if sm_repo.is_dirty(index=deleted or modified or added or type_changed,
                                    working_tree=deleted or modified or added or type_changed,
                                    untracked_files=untracked,
                                    submodules=False, path=path):
                    sm_dirty = True
            else:
                # uninitialized submodule
                # it can't be dirty and we can't recurse any deeper:
                continue

            if sm_dirty:
                # the submodule itself is dirty
                modified_subs.append(sm.path)
            else:
                # the submodule itself is clean, recurse:
                # TODO: This fails ATM with AttributeError, if sm is a GitRepo.
                # we need get_status and this recursion method to be available
                # to both classes. Issue: We need to be able to come back to
                # AnnexRepo from GitRepo if there's again an annex beneath. But
                # we can't import AnnexRepo in gitrepo.py.
                modified_subs.extend(
                    sm_repo._submodules_dirty_direct_mode(
                        untracked=untracked, deleted=deleted,
                        modified=modified, added=added,
                        type_changed=type_changed, path=path
                    ))

        return modified_subs

    def get_status(self, untracked=True, deleted=True, modified=True, added=True,
                   type_changed=True, submodules=True, path=None):
        """Return various aspects of the status of the annex repository

        Note: Under certain circumstances newly added submodules might be
        reported as 'modified' rather tha 'added'.
        See `AnnexRepo._submodules_dirty_direct_mode` for details.

        Parameters
        ----------
        untracked
        deleted
        modified
        added
        type_changed
        submodules
        path

        Returns
        -------

        """

        self.precommit()

        options = assure_list(path) if path else []
        if not submodules:
            options.extend(to_options(ignore_submodules='all'))

        # BEGIN workaround bug (see self._submodules_dirty_direct_mode)
        # internal call to 'git status' by 'git annex status' will fail
        # in submodules without a working tree (direct mode)
        # How to catch this case depends on annex version, since annex
        # exits zero until version 6.20170307

        def _fake_exception_wrapper(self, options_):
            """generate a faked `CommandError` from logged stderr output"""

            # this is for use with older annex, which didn't exit non-zero
            # in case of the failure we are interested in

            # TODO: If we are to keep this workaround (we probably rely on a
            # newer annex anyway), we should not use swallow_logs, since we
            # actually don't want to swallow it, but inspect it. Use a proper
            # handler/filter for the logger instead to not create temp files via
            # swallow_logs

            old_log_state = self.cmd_call_wrapper.log_outputs
            self.cmd_call_wrapper._log_opts['outputs'] = True

            with swallow_logs(new_level=logging.ERROR) as cml:
                # Note, that _run_annex_command_json returns a generator
                json_list = \
                    list(self._run_annex_command_json(
                        'status', opts=options_, expect_stderr=False))
            self.cmd_call_wrapper._log_opts['outputs'] = old_log_state
            if "fatal:" in cml.out:
                raise CommandError(cmd="git annex status",
                                   msg=cml.out, stderr=cml.out)
            return json_list

        try:
            if self.git_annex_version < '6.20170307':
                json_list = _fake_exception_wrapper(self, options_=options)
            else:
                json_list = \
                    list(self._run_annex_command_json(
                        'status', opts=options, expect_stderr=False))
        except CommandError as e:
            if submodules and AnnexRepo._is_annex_work_tree_message(e.stderr):
                lgr.debug("git-annex-status failed probably due to submodule in"
                          " direct mode. Trying to workaround.")
                # try again, ignoring submodules:
                options = [path] if path else []
                options.extend(to_options(ignore_submodules='all'))
                json_list = list(
                    self._run_annex_command_json('status', opts=options)
                )
                # separately get modified submodules:
                m_subs = \
                    self._submodules_dirty_direct_mode(untracked=untracked,
                                                       deleted=deleted,
                                                       modified=modified,
                                                       added=added,
                                                       type_changed=type_changed,
                                                       path=path)
                json_list.extend({'file': p, 'status': 'M'} for p in m_subs)
            else:
                # not the known bug we want to catch
                raise e

        # END workaround

        key_mapping = [(untracked, 'untracked', '?'),
                       (deleted, 'deleted', 'D'),
                       (modified, 'modified', 'M'),
                       (added, 'added', 'A'),
                       (type_changed, 'type_changed', 'T')]
        from datalad.utils import with_pathsep
        return {key: [with_pathsep(i['file'])
                      if isdir(opj(self.path, i['file'])) else i['file']
                      # for consistency with 'git status' return directories
                      # with trailing path separator
                      for i in json_list if i['status'] == st]
                for cond, key, st in key_mapping if cond}

    @borrowdoc(GitRepo)
    def is_dirty(self, index=True, working_tree=False, untracked_files=True,
                 submodules=True, path=None):
        # TODO: Add doc on how this differs from GitRepo.is_dirty()
        # Parameter working_tree exists to meet the signature of GitRepo.is_dirty()

        if working_tree:
            # Note: annex repos don't always have a git working tree and the
            # behaviour in direct mode or V6 repos is fundamentally different
            # from that concept. There are no unstaged changes in direct mode
            # for example. Therefore the need to call this method with
            # 'working_tree=True' indicates invalid assumptions in the
            # calling code.

            # TODO: Better exception. InvalidArgumentError or sth ...
            raise CommandNotAvailableError(
                "Querying a git-annex repository for a clean/dirty "
                "working tree is an invalid concept.")
        # Again note, that 'annex status' isn't distinguishing staged and
        # unstaged changes, since this makes little sense for an annex repo
        # in general. Therefore we use only 'index' and 'untracked_files' to
        # specify what kind of dirtyness we are interested in:
        status = self.get_status(untracked=untracked_files, deleted=index,
                                 modified=index, added=index,
                                 type_changed=index, submodules=submodules,
                                 path=path)
        return any([bool(status[i]) for i in status])

    @property
    def untracked_files(self):
        """Get a list of untracked files
        """
        return self.get_status(untracked=True, deleted=False, modified=False,
                               added=False, type_changed=False, submodules=False,
                               path=None)['untracked']

    @classmethod
    def _check_git_annex_version(cls):
        ver = external_versions['cmd:annex']
        # in case it is missing
        if linux_distribution_name in {'debian', 'ubuntu'}:
            msg = "Install  git-annex-standalone  from NeuroDebian " \
                  "(http://neuro.debian.net)"
        else:
            msg = "Visit http://git-annex.branchable.com/install/"
        exc_kwargs = dict(
            name="git-annex",
            msg=msg,
            ver=cls.GIT_ANNEX_MIN_VERSION
        )
        if not ver:
            raise MissingExternalDependency(**exc_kwargs)
        elif ver < cls.GIT_ANNEX_MIN_VERSION:
            raise OutdatedExternalDependency(ver_present=ver, **exc_kwargs)
        cls.git_annex_version = ver

    @staticmethod
    def get_size_from_key(key):
        """A little helper to obtain size encoded in a key"""
        if not key:
            return None
        try:
            size_str = key.split('-', 2)[1].lstrip('s')
        except IndexError:
            # has no 2nd field in the key
            return None
        return int(size_str) if size_str.isdigit() else None

    @normalize_path
    def get_file_size(self, path):
        fpath = opj(self.path, path)
        return 0 if not exists(fpath) else os.stat(fpath).st_size

    # TODO: Once the PR containing super class 'Repo' was merged, move there and
    # melt with GitRepo.get_toppath including tests for both
    @classmethod
    def get_toppath(cls, path, follow_up=True, git_options=None):
        """Return top-level of a repository given the path.

        Parameters
        -----------
        follow_up : bool
          If path has symlinks -- they get resolved by git.  If follow_up is
          True, we will follow original path up until we hit the same resolved
          path.  If no such path found, resolved one would be returned.
        git_options: list of str
          options to be passed to the git rev-parse call

        Return None if no parent directory contains a git repository.
        """

        # first try plain git result:
        toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,
                                      git_options=git_options)
        if toppath == '':
            # didn't fail, so git itself didn't come to the conclusion
            # there is no repo, but we have no actual result;
            # might be an annex in direct mode
            if git_options is None:
                git_options = []
            # TODO: Apparently doesn't work with git 2.11.0
            # Note: Since we are in a classmethod, GitRepo.get_toppath uses
            # Runner directly instead of _git_custom_command, which is why the
            # common mechanics for direct mode are not applied.
            # This is why there is no solution for git 2.11 yet

            # Note 2: Actually, the above issue is irrelevant. The git
            # executable has no repository it is bound to, since it's the
            # purpose of the call to find this repository. Therefore
            # core.bare=False has no effect at all.

            # Disabled. See notes.
            # git_options.extend(['-c', 'core.bare=False'])
            # toppath = GitRepo.get_toppath(path=path, follow_up=follow_up,
            #                               git_options=git_options)

            # basically a copy of code in GitRepo.get_toppath
            # except it uses 'git rev-parse --git-dir' as a workaround for
            # direct mode:

            from os.path import dirname
            from os import pardir

            cmd = ['git']
            if git_options:
                cmd.extend(git_options)

            cmd.append("rev-parse")
            if external_versions['cmd:git'] >= '2.13.0':
                cmd.append("--absolute-git-dir")
            else:
                cmd.append("--git-dir")

            try:
                toppath, err = GitRunner().run(
                    cmd,
                    cwd=path,
                    log_stdout=True, log_stderr=True,
                    expect_fail=True, expect_stderr=True)
                toppath = toppath.rstrip('\n\r')
            except CommandError:
                return None
            except OSError:
                toppath = AnnexRepo.get_toppath(dirname(path),
                                                follow_up=follow_up,
                                                git_options=git_options)

            if external_versions['cmd:git'] < '2.13.0':
                # we got a path relative to `path` instead of an absolute one
                toppath = opj(path, toppath)

            # we got the git-dir. Assuming the root dir we are looking for is
            # one level up:
            toppath = realpath(normpath(opj(toppath, pardir)))

            if follow_up:
                path_ = path
                path_prev = ""
                while path_ and path_ != path_prev:  # on top /.. = /
                    if realpath(path_) == toppath:
                        toppath = path_
                        break
                    path_prev = path_
                    path_ = dirname(path_)

        return toppath

    @classmethod
    def is_valid_repo(cls, path, allow_noninitialized=False):
        """Return True if given path points to an annex repository
        """
        # Note: default value for allow_noninitialized=False is important
        # for invalidating an instance via self._flyweight_invalid. If this is
        # changed, we also need to override _flyweight_invalid and explicitly
        # pass allow_noninitialized=False!

        def git_file_has_annex(p):
            """Return True if `p` contains a .git file, that points to a git
            dir with a subdir 'annex'"""
            _git = opj(p, '.git')
            if not os.path.isfile(_git):
                return False
            with open(_git, "r") as f:
                line = f.readline()
                if line.startswith("gitdir: "):
                    return exists(opj(p, line[8:], 'annex'))
                else:
                    lgr.debug("Invalid .git file: %s", _git)
                    return False

        initialized_annex = GitRepo.is_valid_repo(path) and \
            (exists(opj(path, '.git', 'annex')) or
             git_file_has_annex(path))

        if allow_noninitialized:
            try:
                return initialized_annex \
                    or GitRepo(path, create=False, init=False).is_with_annex()
            except (NoSuchPathError, InvalidGitRepositoryError):
                return False
        else:
            return initialized_annex

    def add_remote(self, name, url, options=None):
        """Overrides method from GitRepo in order to set
        remote.<name>.annex-ssh-options in case of a SSH remote."""
        super(AnnexRepo, self).add_remote(name, url, options if options else [])
        self._set_shared_connection(name, url)

    def set_remote_url(self, name, url, push=False):
        """Overrides method from GitRepo in order to set
        remote.<name>.annex-ssh-options in case of a SSH remote."""

        super(AnnexRepo, self).set_remote_url(name, url, push=push)
        self._set_shared_connection(name, url)

    def set_remote_dead(self, name):
        """Announce to annex that remote is "dead"
        """
        return self._annex_custom_command([], ["git", "annex", "dead", name])

    def is_remote_annex_ignored(self, remote):
        """Return True if remote is explicitly ignored"""
        return self.config.getbool(
            'remote.{}'.format(remote), 'annex-ignore',
            default=False
        )

    def is_special_annex_remote(self, remote, check_if_known=True):
        """Return either remote is a special annex remote

        Decides based on the presence of diagnostic annex- options
        for the remote
        """
        if check_if_known:
            if remote not in self.get_remotes():
                raise RemoteNotAvailableError(remote)
        sec = 'remote.{}'.format(remote)
        for opt in ('annex-externaltype', 'annex-webdav'):
            if self.config.has_option(sec, opt):
                return True
        return False

    @borrowkwargs(GitRepo)
    def get_remotes(self,
                    with_urls_only=False,
                    exclude_special_remotes=False):
        """Get known (special-) remotes of the repository

        Parameters
        ----------
        exclude_special_remotes: bool, optional
          if True, don't return annex special remotes

        Returns
        -------
        remotes : list of str
          List of names of the remotes
        """
        remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only)

        if exclude_special_remotes:
            return [
                remote for remote in remotes
                if not self.is_special_annex_remote(remote, check_if_known=False)
            ]
        else:
            return remotes

    def get_special_remotes(self):
        """Get info about all known (not just enabled) special remotes.

        Returns
        -------
        dict
          Keys are special remore UUIDs, values are dicts with arguments
          for `git-annex enableremote`. This includes at least the 'type'
          and 'name' of a special remote. Each type of special remote
          may require addition arguments that will be available in the
          respective dictionary.
        """
        try:
            stdout, stderr = self._git_custom_command(
                None, ['git', 'cat-file', 'blob', 'git-annex:remote.log'],
                expect_fail=True)
        except CommandError as e:
            if 'Not a valid object name git-annex:remote.log' in e.stderr:
                # no special remotes configures
                return {}
            else:
                # some unforseen error
                raise e
        argspec = re.compile(r'^([^=]*)=(.*)$')
        srs = {}
        for line in stdout.splitlines():
            # be precise and split by spaces
            fields = line.split(' ')
            # special remote UUID
            sr_id = fields[0]
            # the rest are config args for enableremote
            sr_info = dict(argspec.match(arg).groups()[:2] for arg in fields[1:])
            srs[sr_id] = sr_info
        return srs

    def __repr__(self):
        return "<AnnexRepo path=%s (%s)>" % (self.path, type(self))

    def _run_annex_command(self, annex_cmd,
                           git_options=None, annex_options=None,
                           backend=None, jobs=None,
                           files=None,
                           **kwargs):
        """Helper to run actual git-annex calls

        Unifies annex command calls.

        Parameters
        ----------
        annex_cmd: str
            the actual git-annex command, like 'init' or 'add'
        git_options: list of str
            options to be passed to git
        annex_options: list of str
            options to be passed to the git-annex command
        backend: str
            backend to be used by this command; Currently this can also be
            achieved by having an item '--backend=XXX' in annex_options.
            This may change.
        jobs : int
        files: list, optional
            If command passes list of files
        **kwargs
            these are passed as additional kwargs to datalad.cmd.Runner.run()

        Raises
        ------
        CommandNotAvailableError
            if an annex command call returns "unknown command"
        """
        debug = ['--debug'] if lgr.getEffectiveLevel() <= 8 else []
        backend = ['--backend=%s' % backend] if backend else []

        git_options = (git_options[:] if git_options else []) + self._ANNEX_GIT_COMMON_OPTIONS
        annex_options = annex_options[:] if annex_options else []
        if self._annex_common_options:
            annex_options = self._annex_common_options + annex_options

        if not self.always_commit:
            git_options += ['-c', 'annex.alwayscommit=false']

        if git_options:
            cmd_list = ['git'] + git_options + ['annex']
        else:
            cmd_list = ['git-annex']
        if jobs:
            annex_options += ['-J%d' % jobs]

        cmd_list += [annex_cmd] + backend + debug + annex_options

        env = kwargs.pop("env", None)
        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        if files:
            cmd_list += ['--'] + files
        try:
            return self.cmd_call_wrapper.run(cmd_list, env=env, **kwargs)
        except CommandError as e:
            if e.stderr and "git-annex: Unknown command '%s'" % annex_cmd in e.stderr:
                raise CommandNotAvailableError(str(cmd_list),
                                               "Unknown command:"
                                               " 'git-annex %s'" % annex_cmd,
                                               e.code, e.stdout, e.stderr)
            else:
                raise e

    def _run_simple_annex_command(self, *args, **kwargs):
        """Run an annex command and return its output, of which expect 1 line

        Just a little helper to interact with basic annex commands and process
        their output while ignoring some messages

        Parameters
        ----------
        **kwargs: all passed into _run
        """
        out, err = self._run_annex_command(
            *args, **kwargs
        )
        lines = out.rstrip('\n').splitlines()
        # ignore some lines which might appear on a fresh clone
        # see https://git-annex.branchable.com/todo/output_of_wanted___40__and_possibly_group_etc__41___should_not_be_polluted_with___34__informational__34___messages/
        lines_ = [
            l for l in lines
            if not re.search(
                '\((merging .* into git-annex|recording state ).*\.\.\.\)', l
            )
        ]
        assert(len(lines_) <= 1)
        return lines_[0] if lines_ else None

    def _is_direct_mode_from_config(self):
        """Figure out if in direct mode from the git config.

        Since relies on reading config, expensive to be used often

        Returns
        -------
        True if in direct mode, False otherwise.
        """
        # If .git/config lacks an entry "direct",
        # it's actually indirect mode.
        self.config.reload()
        return self.config.getbool("annex", "direct", False)

    def is_direct_mode(self):
        """Return True if annex is in direct mode

        Returns
        -------
        True if in direct mode, False otherwise.
        """
        # TEMP: Disable lazy loading and make sure to read from file every time
        # instead, since we might have several instances pointing to the very
        # same repo atm. TODO: We can remove that, right?
        self.repo.config_reader()._is_initialized = False
        self.repo.config_reader().read()
        self._direct_mode = None

        if self._direct_mode is None:
            # we need to figure it out
            self._direct_mode = self._is_direct_mode_from_config()
        return self._direct_mode

    def is_crippled_fs(self):
        """Return True if git-annex considers current filesystem 'crippled'.

        Returns
        -------
        True if on crippled filesystem, False otherwise
        """

        self.config.reload()
        return self.config.getbool("annex", "crippledfilesystem", False)

    def set_direct_mode(self, enable_direct_mode=True):
        """Switch to direct or indirect mode

        Parameters
        ----------
        enable_direct_mode: bool
            True means switch to direct mode,
            False switches to indirect mode

        Raises
        ------
        CommandNotAvailableError
            in case you try to switch to indirect mode on a crippled filesystem
        """
        if self.is_crippled_fs() and not enable_direct_mode:
            raise CommandNotAvailableError(
                cmd="git-annex indirect",
                msg="Can't switch to indirect mode on that filesystem.")

        self._run_annex_command('direct' if enable_direct_mode else 'indirect',
                                expect_stderr=True)
        self.config.reload()

        # For paranoid we will just re-request
        self._direct_mode = None
        assert(self.is_direct_mode() == enable_direct_mode)

        if self.is_direct_mode():
            # adjust git options for plain git calls on this repo:
            # Note: Not sure yet, whether this solves the issue entirely or we
            # still need 'annex proxy' in some cases ...

            lgr.debug("detected git version: %s" % external_versions['cmd:git'])

            if external_versions['cmd:git'] >= '2.9.0':
                # workaround for git 2.9.0, which for some reason ignores the
                # per-call config "-c core.bare=False", but respects the value
                # if it is set in .git/config
                self.GIT_DIRECT_MODE_WRAPPER_ACTIVE = True

            # TEMP: nevertheless use this option to inject it into gitpython
            # TODO: Solve it and change to "elif"
            if 'core.bare=False' not in self._GIT_COMMON_OPTIONS:
                # standard direct mode procedure part I:
                self._GIT_COMMON_OPTIONS.extend(['-c', 'core.bare=False'])
            if '--work-tree=' not in self._GIT_COMMON_OPTIONS:
                # standard direct mode procedure part II:
                self._GIT_COMMON_OPTIONS.append('--work-tree=.')

    def _git_custom_command(self, *args, **kwargs):

        if self.GIT_DIRECT_MODE_PROXY:
            proxy_str = "git annex proxy -- "
            proxy_list = ['git', 'annex', 'proxy', '--']
            cmd = kwargs.pop("cmd_str", None)
            if not cmd:
                cmd = args[1]
            assert(cmd is not None)

            if isinstance(cmd, string_types):
                cmd = proxy_str + cmd
            else:
                cmd = proxy_list + cmd

            args = (args[0], cmd) + args[2:]
            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

        elif self.GIT_DIRECT_MODE_WRAPPER_ACTIVE:
            old = self.config.get('core.bare')
            lgr.debug("old config: %s(%s)" % (old, type(old)))
            if old is not False:
                self.config.set('core.bare', 'False', where='local')

            out, err = super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

            if old is None:
                self.config.unset('core.bare', where='local')
            elif old:
                self.config.set('core.bare', old, where='local')
            return out, err

        else:
            return super(AnnexRepo, self)._git_custom_command(*args, **kwargs)

    def _init(self, version=None, description=None):
        """Initializes an annex repository.

        Note: This is intended for private use in this class by now.
        If you have an object of this class already,
        there shouldn't be a need to 'init' again.

        """
        # MIH: this function is required for re-initing repos. The logic
        # in the constructor is rather convoluted and doesn't acknowledge
        # the case of a perfectly healthy annex that just needs a new
        # description
        # will keep leading underscore in the name for know, but this is
        # not private
        # TODO: provide git and git-annex options.
        # TODO: Document (or implement respectively) behaviour in special cases
        # like direct mode (if it's different), not existing paths, etc.
        opts = []
        if description is not None:
            opts += [description]
        if version is not None:
            opts += ['--version', '{0}'.format(version)]
        if not len(opts):
            opts = None

        # Note: git-annex-init kills a possible tracking branch for
        # 'annex/direct/my_branch', if we just cloned from a repo in direct
        # mode. We want to preserve the information about the tracking branch,
        # as if the source repo wasn't in direct mode.
        # Note 2: Actually we do it for all 'managed branches'. This might turn
        # out to not be necessary
        sections_to_preserve = ["branch.{}".format(branch)
                                for branch in self.get_branches()
                                if self.is_managed_branch(branch)
                                and "branch.{}".format(branch) in
                                self.config.sections()]
        for sct in sections_to_preserve:
            orig_branch = sct[7:]
            new_branch = self.get_corresponding_branch(orig_branch)
            new_section = "branch.{}".format(new_branch)
            for opt in self.config.options(sct):
                orig_value = self.config.get_value(sct, opt)
                new_value = orig_value.replace(orig_branch, new_branch)
                self.config.add(var=new_section + "." + opt,
                                value=new_value,
                                where='local',
                                reload=False)
        self._run_annex_command(
            'init',
            log_stderr=lambda s: lgr.info(s.rstrip()), log_online=True,
            annex_options=opts)
        # TODO: When to expect stderr?
        # on crippled filesystem for example (think so)?
        self.config.reload()

    @normalize_paths
    def get(self, files, remote=None, options=None, jobs=None, key=False):
        """Get the actual content of files

        Parameters
        ----------
        files : list of str
            paths to get
        remote : str, optional
            from which remote to fetch content
        options : list of str, optional
            commandline options for the git annex get command
        jobs : int or None, optional
            how many jobs to run in parallel (passed to git-annex call).
            If not specified (None), then
        key : bool, optional
            If provided file value is actually a key

        Returns
        -------
        files : list of dict
        """
        options = options[:] if options else []

        if remote:
            if remote not in self.get_remotes():
                raise RemoteNotAvailableError(
                    remote=remote,
                    cmd="get",
                    msg="Remote is not known. Known are: %s"
                    % (self.get_remotes(),)
                )
            options += ['--from', remote]

        # analyze provided files to decide which actually are needed to be
        # fetched

        if not key:
            expected_downloads, fetch_files = self._get_expected_files(
                files, ['--not', '--in', 'here'])
        else:
            fetch_files = files
            assert len(files) == 1, "When key=True only a single file be provided"
            expected_downloads = {files[0]: AnnexRepo.get_size_from_key(files[0])}

        if not fetch_files:
            lgr.debug("No files found needing fetching.")
            return []

        if len(fetch_files) != len(files):
            lgr.debug("Actually getting %d files", len(fetch_files))

        # TODO: provide more meaningful message (possibly aggregating 'note'
        #  from annex failed ones
        # TODO: reproduce DK's bug on OSX, and either switch to
        #  --batch mode (I don't think we have --progress support in long
        #  alive batch processes ATM),
        if key:
            kwargs = {'opts': options + ['--key'] + files}
        else:
            kwargs = {'opts': options, 'files': files}
        results = self._run_annex_command_json(
            'get',
            # TODO: eventually make use of --batch mode
            jobs=jobs,
            expected_entries=expected_downloads,
            progress=True,
            **kwargs
        )
        results_list = list(results)
        # TODO:  should we here compare fetch_files against result_list
        # and vomit an exception of incomplete download????
        return results_list

    def _get_expected_files(self, files, expr):
        """Given a list of files, figure out what to be downloaded

        Parameters
        ----------
        files
        expr: list
          Expression to be passed into annex's find

        Returns
        -------
        expected_files : dict
          key -> size
        fetch_files : list
          files to be fetched
        """
        lgr.debug("Determine what files match the query to work with")
        # Let's figure out first which files/keys and of what size to download
        expected_files = {}
        fetch_files = []
        keys_seen = set()
        unknown_sizes = []  # unused atm
        # for now just record total size, and
        for j in self._run_annex_command_json(
                'find', opts=expr, files=files
        ):
            # TODO: some files might not even be here.  So in current fancy
            # output reporting scheme we should then theoretically handle
            # those cases here and say 'impossible' or something like that
            if not j.get('success', True):
                # TODO: I guess do something with yielding and filtering for
                # what need to be done and what not
                continue
            key = j['key']
            size = j.get('bytesize')
            if key in keys_seen:
                # multiple files could point to the same key.  no need to
                # request multiple times
                continue
            keys_seen.add(key)
            assert j['file']
            fetch_files.append(j['file'])
            if size and size.isdigit():
                expected_files[key] = int(size)
            else:
                expected_files[key] = None
                unknown_sizes.append(j['file'])
        return expected_files, fetch_files

    @normalize_paths
    def add(self, files, git=None, backend=None, options=None, jobs=None,
            git_options=None, annex_options=None, update=False):
        """Add file(s) to the repository.

        Parameters
        ----------
        files: list of str
          list of paths to add to the annex
        git: bool
          if True, add to git instead of annex.
        backend:
        options:
        update: bool
          --update option for git-add. From git's manpage:
           Update the index just where it already has an entry matching
           <pathspec>. This removes as well as modifies index entries to match
           the working tree, but adds no new files.

           If no <pathspec> is given when --update option is used, all tracked
           files in the entire working tree are updated (old versions of Git
           used to limit the update to the current directory and its
           subdirectories).

           Note: Used only, if a call to git-add instead of git-annex-add is
           performed

        Returns
        -------
        list of dict
        """

        return list(self.add_(
            files, git=git, backend=backend, options=options, jobs=jobs,
            git_options=git_options, annex_options=annex_options, update=update
        ))

    def add_(self, files, git=None, backend=None, options=None, jobs=None,
            git_options=None, annex_options=None, update=False):
        """Like `add`, but returns a generator"""
        if update and not git:
            raise InsufficientArgumentsError("option 'update' requires 'git', too")

        if git_options:
            # TODO: note that below we would use 'add with --dry-run
            # so passed here options might need to be passed into it??
            lgr.warning("add: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []
        # Note: As long as we support direct mode, one should not call
        # super().add() directly. Once direct mode is gone, we might remove
        # `git` parameter and call GitRepo's add() instead.

        def _get_to_be_added_recs(paths):
            """Try to collect what actually is going to be added

            This is used for progress information
            """

            if self.is_direct_mode():
                # we already know we can't use --dry-run
                for r in self._process_git_get_output(
                        linesep.join(["'{}'".format(p.encode('utf-8'))
                        for p in paths])):
                    yield r
                    return
            else:
                # Note: if a path involves a submodule in direct mode, while we
                # are not in direct mode at current level, we might still fail.
                # Hence the except clause is still needed. However, this is
                # unlikely, since direct mode usually should be used only, if it
                # was enforced by FS and/or OS and therefore concerns the entire
                # hierarchy.
                _git_options = ['--dry-run', '-N', '--ignore-missing']
                try:
                    for r in super(AnnexRepo, self).add_(
                            files, git_options=_git_options, update=update):
                        yield r
                        return
                except CommandError as e:
                    if AnnexRepo._is_annex_work_tree_message(e.stderr):
                        lgr.warning(
                            "Known bug in direct mode."
                            "We can't use --dry-run when there are submodules in "
                            "direct mode, because the internal call to git status "
                            "fails. To be resolved by using (Dataset's) status "
                            "instead of a git-add --dry-run altogether.")
                        # fake the return for now
                        for r in self._process_git_get_output(
                                linesep.join(["'{}'".format(f)
                                for f in files])):
                            yield r
                            return
                    else:
                        # unexpected failure
                        raise e

        # Theoretically we could have done for git as well, if it could have
        # been batched
        # Call git annex add for any to have full control of either to go
        # to git or to annex
        # 1. Figure out what actually will be added
        to_be_added_recs = _get_to_be_added_recs(files)
        # collect their sizes for the progressbar
        expected_additions = {
            rec['file']: self.get_file_size(rec['file'])
            for rec in to_be_added_recs
        }

        # if None -- leave it to annex to decide
        if git is not None:
            options += [
                '-c',
                'annex.largefiles=%s' % (('anything', 'nothing')[int(git)])
            ]
            if git:
                # to maintain behaviour similar to git
                options += ['--include-dotfiles']

        if git and update:
            # explicitly use git-add with --update instead of git-annex-add
            # TODO: This might still need some work, when --update AND files
            # are specified!
            if self.is_direct_mode() and not files:
                self.GIT_DIRECT_MODE_PROXY = True
            try:
                for r in super(AnnexRepo, self).add(
                        files,
                        git=True,
                        git_options=git_options,
                        update=update):
                    yield r
            finally:
                if self.is_direct_mode() and not files:
                    # don't accidentally cause other git calls to be done
                    # via annex-proxy
                    self.GIT_DIRECT_MODE_PROXY = False

        else:
            for r in self._run_annex_command_json(
                    'add',
                    opts=options,
                    files=files,
                    backend=backend,
                    expect_fail=True,
                    jobs=jobs,
                    expected_entries=expected_additions,
                    expect_stderr=True):
                yield r

    def proxy(self, git_cmd, **kwargs):
        """Use git-annex as a proxy to git

        This is needed in case we are in direct mode, since there's no git
        working tree, that git can handle.

        Parameters
        ----------
        git_cmd: list of str
            the actual git command
        `**kwargs`: dict, optional
            passed to _run_annex_command

        Returns
        -------
        (stdout, stderr)
            output of the command call
        """
        # TODO: We probably don't need it anymore

        if not self.is_direct_mode():
            lgr.warning("proxy() called in indirect mode: %s" % git_cmd)
            raise CommandNotAvailableError(cmd="git annex proxy",
                                           msg="Proxy doesn't make sense"
                                               " if not in direct mode.")
        return self._run_annex_command('proxy',
                                       annex_options=['--'] + git_cmd,
                                       **kwargs)

    @normalize_paths
    def get_file_key(self, files):
        """Get key of an annexed file.

        Parameters
        ----------
        files: str or list
            file(s) to look up

        Returns
        -------
        str or list
            keys used by git-annex for each of the files;
            in case of a list an empty string is returned if there was no key
            for that file
        """

        if len(files) > 1:
            return self._batched.get('lookupkey', path=self.path)(files)
        else:
            files = files[0]
            # single file
            # keep current implementation
            # TODO: This should change, but involves more RF'ing and an
            # alternative regarding FileNotInAnnexError
            cmd_str = 'git annex lookupkey %s' % files  # have a string for messages

            try:
                out, err = self._run_annex_command(
                    'lookupkey',
                    files=[files],
                    expect_fail=True
                )
            except CommandError as e:
                if e.code == 1:
                    if not exists(opj(self.path, files)):
                        raise IOError(e.code, "File not found.", files)
                    # XXX you don't like me because I can be real slow!
                    elif files in self.get_indexed_files():
                        # if we got here, the file is present and in git,
                        # but not in the annex
                        raise FileInGitError(cmd=cmd_str,
                                             msg="File not in annex, but git: %s"
                                                 % files,
                                             filename=files)
                    else:
                        raise FileNotInAnnexError(cmd=cmd_str,
                                                  msg="File not in annex: %s"
                                                      % files,
                                                  filename=files)
                else:
                    # Not sure, whether or not this can actually happen
                    raise e

            entries = out.rstrip(linesep).splitlines()
            # filter out the ones which start with (: http://git-annex.branchable.com/bugs/lookupkey_started_to_spit_out___34__debug__34___messages_to_stdout/?updated
            entries = list(filter(lambda x: not x.startswith('('), entries))
            if len(entries) > 1:
                lgr.warning("Got multiple entries in reply asking for a key of a file: %s"
                            % (str(entries)))
            elif not entries:
                raise FileNotInAnnexError("Could not get a key for a file(s) %s -- empty output" % files)
            return entries[0]

    @normalize_paths
    def lock(self, files, options=None):
        """undo unlock

        Use  this to undo an unlock command if you don't want to modify the
        files any longer, or have made modifications you want to discard.

        Parameters
        ----------
        files: list of str
        options: list of str
        """

        options = options[:] if options else []
        self._run_annex_command('lock', annex_options=options, files=files)
        # note: there seems to be no output by annex if success.

    @normalize_paths
    def unlock(self, files, options=None):
        """unlock files for modification

        Parameters
        ----------
        files: list of str
        options: list of str

        Returns
        -------
        list of str
          successfully unlocked files
        """

        options = options[:] if options else []

        if self.is_direct_mode():

            # TODO:
            # If anything there should be a CommandNotAvailableError now:
            lgr.debug("'%s' is in direct mode, "
                      "'annex unlock' not available", self)
            lgr.warning("In direct mode there is no 'unlock'. However if "
                        "the file's content is present, it is kind of "
                        "unlocked. Therefore just checking whether this is "
                        "the case.")
            # TODO/FIXME:
            # Note: the following isn't exactly nice, if `files` is a dir.
            # For a "correct" result we would need to report all files within
            # potential dir(s) in `files`, that are annexed and have content.
            # Also note, that even now files in git might be reported "unlocked",
            # since they have content. This might be a confusing result.
            # On the other hand, this is solved on the level of Dataset.unlock
            # by annotating those paths 'notneeded' beforehand.
            return [f for f in files if self.file_has_content(f)]

        else:

            # TODO: catch and parse output if failed (missing content ...)
            std_out, std_err = \
                self._run_annex_command(
                    'unlock', annex_options=options, files=files
                )

            return [line.split()[1]
                    for line in std_out.splitlines()
                    if line.split()[0] == 'unlock' and line.split()[-1] == 'ok']

    def adjust(self, options=None):
        """enter an adjusted branch

        This command is only available in a v6 git-annex repository.

        Parameters
        ----------
        options: list of str
          currently requires '--unlock' or '--fix';
          default: --unlock
        """
        # TODO: Do we want to catch the case that
        # "adjusted/<current_branch_name>(unlocked)" already exists and
        # just check it out? Or fail like annex itself does?

        # version check:
        if not self.config.get("annex.version") == '6':
            raise CommandNotAvailableError(cmd='git annex adjust',
                                           msg='git-annex-adjust requires a '
                                               'version 6 repository')

        options = options[:] if options else to_options(unlock=True)
        self._run_annex_command('adjust', annex_options=options)

    @normalize_paths
    def unannex(self, files, options=None):
        """undo accidental add command

        Use this to undo an accidental git annex add command. Note that for
        safety, the content of the file remains in the annex, until you use git
        annex unused and git annex dropunused.

        Parameters
        ----------
        files: list of str
        options: list of str

        Returns
        -------
        list of str
          successfully unannexed files
        """

        options = options[:] if options else []

        std_out, std_err = self._run_annex_command(
            'unannex', annex_options=options, files=files
        )
        return [line.split()[1] for line in std_out.splitlines()
                if line.split()[0] == 'unannex' and line.split()[-1] == 'ok']

    @normalize_paths(map_filenames_back=True)
    def find(self, files, batch=False):
        """Run `git annex find` on file(s).

        Parameters
        ----------
        files: list of str
            files to find under annex
        batch: bool, optional
            initiate or continue with a batched run of annex find, instead of just
            calling a single git annex find command. If any items in `files`
            are directories, this value is treated as False.

        Returns
        -------
        A dictionary the maps each item in `files` to its `git annex find`
        result. Items without a successful result will be an empty string, and
        multi-item results (which can occur for if `files` includes a
        directory) will be returned as a list.
        """
        objects = {}
        # Ignore batch=True if any path is a directory because `git annex find
        # --batch` always returns an empty string for directories.
        if batch and not any(isdir(opj(self.path, f)) for f in files):
            find = self._batched.get('find', json=True, path=self.path)
            objects = {f: json_out.get("file", "")
                       for f, json_out in zip(files, find(files))}
        else:
            for f in files:
                try:
                    obj, er = self._run_annex_command(
                        'find', files=[f],
                        annex_options=["--print0"],
                        expect_fail=True
                    )
                    items = obj.rstrip("\0").split("\0")
                    objects[f] = items[0] if len(items) == 1 else items
                except CommandError:
                    objects[f] = ''

        return objects

    def _check_files(self, fn, quick_fn, files, allow_quick, batch):
        # Helper that isolates the common logic in `file_has_content` and
        # `is_under_annex`. `fn` is the annex command used to do the check, and
        # `quick_fn` is the non-annex variant.
        is_v6 = self.config.get("annex.version") == "6"
        if is_v6 or self.is_direct_mode() or batch or not allow_quick:
            # We're only concerned about modified files in V6 mode. In V5
            # `find` returns an empty string for unlocked files, and in direct
            # mode everything looks modified, so we don't even bother.
            modified = self.get_changed_files() if is_v6 else []
            annex_res = fn(files, normalize_paths=False, batch=batch)
            return [bool(annex_res.get(f) and
                         not (is_v6 and normpath(f) in modified))
                    for f in files]
        else:  # ad-hoc check which should be faster than call into annex
            return [quick_fn(f) for f in files]

    @normalize_paths
    def file_has_content(self, files, allow_quick=True, batch=False):
        """Check whether files have their content present under annex.

        Parameters
        ----------
        files: list of str
            file(s) to check for being actually present.
        allow_quick: bool, optional
            allow quick check, based on having a symlink into .git/annex/objects.
            Works only in non-direct mode (TODO: thin mode)

        Returns
        -------
        list of bool
            For each input file states either file has content locally
        """
        # TODO: Also provide option to look for key instead of path

        def quick_check(filename):
            filepath = opj(self.path, filename)
            if islink(filepath):                    # if symlink
                # find abspath of node pointed to by symlink
                target_path = realpath(filepath)  # realpath OK
                # TODO: checks for being not outside of this repository
                # Note: ben removed '.git/' from '.git/annex/objects',
                # since it is not true for submodules, whose '.git' is a
                # symlink and being resolved to some
                # '.git/modules/.../annex/objects'
                return exists(target_path) and 'annex/objects' in str(target_path)
            return False

        return self._check_files(self.find, quick_check,
                                 files, allow_quick, batch)

    @normalize_paths
    def is_under_annex(self, files, allow_quick=True, batch=False):
        """Check whether files are under annex control

        Parameters
        ----------
        files: list of str
            file(s) to check for being under annex
        allow_quick: bool, optional
            allow quick check, based on having a symlink into .git/annex/objects.
            Works only in non-direct mode (TODO: thin mode)

        Returns
        -------
        list of bool
            For each input file states either file is under annex
        """
        # theoretically in direct mode files without content would also be
        # broken symlinks on the FSs which support it, but that would complicate
        # the matters

        # This is an ugly hack to prevent files from being treated as
        # remotes by `git annex info`. See annex's `nameToUUID'`.
        files = [opj(curdir, f) for f in files]

        def check(files, **kwargs):
            # Filter out directories because it doesn't make sense to ask if
            # they are under annex control and `info` can only handle
            # non-directories.
            return self.info([f for f in files if not isdir(f)],
                             fast=True, **kwargs)

        def quick_check(filename):
            filepath = opj(self.path, filename)
            # todo checks for being not outside of this repository
            # Note: ben removed '.git/' from '.git/annex/objects',
            # since it is not true for submodules, whose '.git' is a
            # symlink and being resolved to some
            # '.git/modules/.../annex/objects'
            return (
                islink(filepath)
                and 'annex/objects' in str(realpath(filepath))  # realpath OK
            )

        return self._check_files(check, quick_check,
                                 files, allow_quick, batch)

    def init_remote(self, name, options):
        """Creates a new special remote

        Parameters
        ----------
        name: str
            name of the special remote
        """
        # TODO: figure out consistent way for passing options + document

        self._run_annex_command('initremote', annex_options=[name] + options)
        self.config.reload()

    def enable_remote(self, name, env=None):
        """Enables use of an existing special remote

        Parameters
        ----------
        name: str
            name, the special remote was created with
        """

        try:
            self._run_annex_command(
                'enableremote',
                annex_options=[name],
                expect_fail=True,
                log_stderr=True,
                env=env)
        except CommandError as e:
            if re.match(r'.*StatusCodeException.*statusCode = 401', e.stderr):
                raise AccessDeniedError(e.stderr)
            elif 'FailedConnectionException' in e.stderr:
                raise AccessFailedError(e.stderr)
            else:
                raise e
        self.config.reload()

    def merge_annex(self, remote=None):
        """Merge git-annex branch

        Merely calls `sync` with the appropriate arguments.

        Parameters
        ----------
        remote: str, optional
          Name of a remote to be "merged".
        """
        self.sync(
            remotes=remote, push=False, pull=False, commit=False, content=False,
            all=False)

    def sync(self, remotes=None, push=True, pull=True, commit=True,
             content=False, all=False, fast=False):
        """Synchronize local repository with remotes

        Use  this  command  when you want to synchronize the local repository
        with one or more of its remotes. You can specify the remotes (or
        remote groups) to sync with by name; the default if none are specified
        is to sync with all remotes.

        Parameters
        ----------
        remotes: str, list(str), optional
          Name of one or more remotes to be sync'ed.
        push : bool
          By default, git pushes to remotes.
        pull : bool
          By default, git pulls from remotes
        commit : bool
          A commit is done by default. Disable to avoid  committing local
          changes.
        content : bool
          Normally, syncing does not transfer the contents of annexed
          files.  This option causes the content of files in the work tree
          to also be uploaded and downloaded as necessary.
        all : bool
          This option, when combined with `content`, makes all available
          versions of all files be synced, when preferred content settings
          allow
        fast : bool
          Only sync with the remotes with the lowest annex-cost value
          configured
        """
        # In direct mode annex-sync fails, if commit=True
        # apparently sync is calling git status internally, which then fails
        # in the submodule. (As we already know)
        # stdout:
        # commit  add second ok
        # (recording state in git...)
        #
        # failed
        # (recording state in git...)
        #
        # stderr:
        # fatal: This operation must be run in a work tree
        # fatal: 'git status --porcelain' failed in submodule submod
        # git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","-f"] exited 123)
        # fatal: This operation must be run in a work tree
        # fatal: 'git status --porcelain' failed in submodule submod
        # git-annex: user error (xargs ["-0","git","--git-dir=.git","--work-tree=.","--literal-pathspecs","add","-f"] exited 123)

        # TODO: Workaround

        args = []
        args.extend(to_options(push=push, no_push=not push,
                               # means: '--push' if push else '--no-push'
                               pull=pull, no_pull=not pull,
                               commit=commit, no_commit=not commit,
                               content=content, no_content=not content,
                               all=all,
                               fast=fast))
        args.extend(assure_list(remotes))
        self._run_annex_command('sync', annex_options=args)

    @normalize_path
    def add_url_to_file(self, file_, url, options=None, backend=None,
                        batch=False, git_options=None, annex_options=None,
                        unlink_existing=False):
        """Add file from url to the annex.

        Downloads `file` from `url` and add it to the annex.
        If annex knows `file` already,
        records that it can be downloaded from `url`.

        Note: Consider using the higher-level `download_url` instead.

        Parameters
        ----------
        file_: str

        url: str

        options: list
            options to the annex command

        batch: bool, optional
            initiate or continue with a batched run of annex addurl, instead of just
            calling a single git annex addurl command

        unlink_existing: bool, optional
            by default crashes if file already exists and is under git.
            With this flag set to True would first remove it.

        Returns
        -------
        dict
          In batch mode only ATM returns dict representation of json output returned
          by annex
        """

        if git_options:
            lgr.warning("add_url_to_file: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []
        git_options = []
        kwargs = dict(backend=backend)
        if lexists(opj(self.path, file_)) and \
                unlink_existing and \
                not self.is_under_annex(file_):
            # already under git, we can't addurl for under annex
            lgr.warning(
                "File %s:%s is already under git, removing so it could possibly"
                " be added under annex", self, file_
            )
            unlink(opj(self.path, file_))
        if not batch or self.fake_dates_enabled:
            if batch:
                lgr.debug("Not batching addurl call "
                          "because fake dates are enabled")
            files_opt = '--file=%s' % file_
            if PY2:
                files_opt = assure_bytes(files_opt)
            out_json = self._run_annex_command_json(
                'addurl',
                opts=options + [files_opt] + [url],
                progress=True,
                expected_entries={file_: None},
                **kwargs
            )
            if len(out_json) != 1:
                raise AssertionError(
                    "should always be a single-item list, Got: %s"
                    % str(out_json))
            # Make the output's structure match bcmd's.
            out_json = out_json[0]
            # Don't capture stderr, since download progress provided by wget uses
            # stderr.
        else:
            options += ['--with-files']
            if backend:
                options += ['--backend=%s' % backend]
            # Initializes (if necessary) and obtains the batch process
            bcmd = self._batched.get(
                # Since backend will be critical for non-existing files
                'addurl_to_file_backend:%s' % backend,
                annex_cmd='addurl',
                git_options=git_options,
                annex_options=options,  # --raw ?
                path=self.path,
                json=True
            )
            try:
                out_json = bcmd((url, file_))
            except Exception as exc:
                # if isinstance(exc, IOError):
                #     raise
                raise AnnexBatchCommandError(
                    cmd="addurl",
                    msg="Adding url %s to file %s failed due to %s" % (url, file_, exc_str(exc)))
            assert(out_json['command'] == 'addurl')
        if not out_json.get('success', False):
            raise (AnnexBatchCommandError if batch else CommandError)(
                    cmd="addurl",
                    msg="Error, annex reported failure for addurl (url='%s'): %s"
                    % (url, str(out_json)))
        return out_json

    def add_urls(self, urls, options=None, backend=None, cwd=None,
                 jobs=None,
                 git_options=None, annex_options=None):
        """Downloads each url to its own file, which is added to the annex.

        Parameters
        ----------
        urls: list of str

        options: list, optional
            options to the annex command

        cwd: string, optional
            working directory from within which to invoke git-annex
        """

        if git_options:
            lgr.warning("add_urls: git_options not yet implemented. Ignored.")

        if annex_options:
            lgr.warning("annex_options not yet implemented. Ignored.")

        options = options[:] if options else []

        self._run_annex_command('addurl', annex_options=options + urls,
                                backend=backend, log_online=True,
                                log_stderr=False, cwd=cwd)
        # Don't capture stderr, since download progress provided by wget uses
        # stderr.

        # currently simulating similar return value, assuming success
        # for all files:
        # TODO: Make return values consistent across both *Repo classes!
        return [{u'file': f, u'success': True} for f in urls]

    @normalize_path
    def rm_url(self, file_, url):
        """Record that the file is no longer available at the url.

        Parameters
        ----------
        file_: str

        url: str
        """

        self._run_annex_command('rmurl', files=[file_, url])

    @normalize_path
    def get_urls(self, file_, key=False, batch=False):
        """Get URLs for a file/key

        Parameters
        ----------
        file_: str
        key: bool, optional
            Either provided files are actually annex keys
        """
        return self.whereis(file_, output='full', batch=batch)[AnnexRepo.WEB_UUID]['urls']

    @normalize_paths
    def drop(self, files, options=None, key=False, jobs=None):
        """Drops the content of annexed files from this repository.

        Drops only if possible with respect to required minimal number of
        available copies.

        Parameters
        ----------
        files: list of str
            paths to drop
        options : list of str, optional
            commandline options for the git annex drop command
        jobs : int, optional
            how many jobs to run in parallel (passed to git-annex call)

        Returns
        -------
        list(JSON objects)
          'success' item in each object indicates failure/success per file
          path.
        """

        # annex drop takes either files or options
        # --all, --unused, --key, or --incomplete
        # for now, most simple test; to be replaced by a more general solution
        # (exception thrown by _run_annex_command)
        if not files and \
                (not options or
                 not any([o in options for o in
                          ["--all", "--unused", "--key", "--incomplete"]])):
            raise InsufficientArgumentsError("drop() requires at least to "
                                             "specify 'files' or 'options'")

        options = assure_list(options)

        if key:
            # we can't drop multiple in 1 line, and there is no --batch yet, so
            # one at a time
            files = assure_list(files)
            options = options + ['--key']
            res = [
                self._run_annex_command_json(
                    'drop',
                    opts=options + [k],
                    jobs=jobs)
                for k in files
            ]
            # `normalize_paths` ... magic, useful?
            if len(files) == 1:
                return res[0]
            else:
                return res
        else:
            return self._run_annex_command_json(
                'drop',
                opts=options,
                files=files,
                jobs=jobs)

    def drop_key(self, keys, options=None, batch=False):
        """Drops the content of annexed files from this repository referenced by keys

        Dangerous: it drops without checking for required minimal number of
        available copies.

        Parameters
        ----------
        keys: list of str, str

        batch: bool, optional
            initiate or continue with a batched run of annex dropkey, instead of just
            calling a single git annex dropkey command
        """
        keys = [keys] if isinstance(keys, string_types) else keys

        options = options[:] if options else []
        options += ['--force']
        if not batch or self.fake_dates_enabled:
            if batch:
                lgr.debug("Not batching drop_key call "
                          "because fake dates are enabled")
            json_objects = self._run_annex_command_json(
                'dropkey', opts=options, files=keys, expect_stderr=True
            )
        else:
            json_objects = self._batched.get(
                'dropkey',
                annex_options=options, json=True, path=self.path
            )(keys)
        for j in json_objects:
            assert j.get('success', True)

    # TODO: a dedicated unit-test
    def _whereis_json_to_dict(self, j):
        """Convert json record returned by annex whereis --json to our dict representation for it
        """
        assert (j.get('success', True) is True)
        # process 'whereis' containing list of remotes
        remotes = {remote['uuid']: {x: remote.get(x, None)
                                    for x in ('description', 'here', 'urls')
                                    }
                   for remote in j.get('whereis')}
        if self.WEB_UUID in remotes:
            assert(remotes[self.WEB_UUID]['description'] == 'web')
        return remotes

    def _run_annex_command_json(self, command,
                                opts=None,
                                jobs=None,
                                files=None,
                                expected_entries=None,
                                progress=False,
                                **kwargs):
        """Run an annex command with --json and load output results into a tuple of dicts

        Parameters
        ----------
        expected_entries : dict, optional
          If provided `filename/key: size` dictionary, will be used to create
          ProcessAnnexProgressIndicators to display progress
        progress: bool, optional
          Whether to request/handle --json-progress
        """
        progress_indicators = None
        try:
            if expected_entries:
                progress_indicators = ProcessAnnexProgressIndicators(
                    expected=expected_entries
                )
                kwargs = kwargs.copy()
                kwargs.update(dict(
                    log_stdout=progress_indicators,
                    log_stderr='offline',  # False, # to avoid lock down
                    log_online=True
                ))
            # TODO: refactor to account for possible --batch ones
            annex_options = ['--json']
            if progress:
                annex_options += ['--json-progress']

            if jobs == 'auto':
                jobs = N_AUTO_JOBS
            if jobs and jobs != 1:
                annex_options += ['-J%d' % jobs]
            if opts:
                # opts might be the '--key' which should go last
                annex_options += opts

            # TODO: RF to use --batch where possible instead of splitting
            # into multiple invocations
            if not files:
                file_chunks = [[]]
            else:
                files = assure_list(files)
                maxl = max(map(len, files))
                chunk_size = max(
                    1,  # should at least be 1. If blows then - not our fault
                    (CMD_MAX_ARG
                     - len(command)
                     - sum((len(x) + 3) for x in annex_options)
                     - 4   # for '--' below
                     - 50  # for safety since we are to add more options
                    // (maxl + 3)  # +3 for possible quotes and a space
                    )
                )
                file_chunks = generate_chunks(files, chunk_size)
            out, err = "", ""
            for file_chunk in file_chunks:
                out_, err_ = self._run_annex_command(
                    command,
                    annex_options=annex_options + ['--'] + file_chunk,
                    **kwargs)
                out += out_
                err += err_
        except CommandError as e:
            # Note: A call might result in several 'failures', that can be or
            # cannot be handled here. Detection of something, we can deal with,
            # doesn't mean there's nothing else to deal with.

            # OutOfSpaceError:
            # Note:
            # doesn't depend on anything in stdout. Therefore check this before
            # dealing with stdout
            out_of_space_re = re.search(
                "not enough free space, need (.*) more", e.stderr
            )
            if out_of_space_re:
                raise OutOfSpaceError(cmd="annex %s" % command,
                                      sizemore_msg=out_of_space_re.groups()[0])

            # RemoteNotAvailableError:
            remote_na_re = re.search(
                "there is no available git remote named \"(.*)\"", e.stderr
            )
            if remote_na_re:
                raise RemoteNotAvailableError(cmd="annex %s" % command,
                                              remote=remote_na_re.groups()[0])

            # TEMP: Workaround for git-annex bug, where it reports success=True
            # for annex add, while simultaneously complaining, that it is in
            # a submodule:
            # TODO: For now just reraise. But independently on this bug, it
            # makes sense to have an exception for that case
            in_subm_re = re.search(
                "fatal: Pathspec '(.*)' is in submodule '(.*)'", e.stderr
            )
            if in_subm_re:
                raise e

            # Note: try to approach the covering of potential annex failures
            # in a more general way:
            # first check stdout:
            if all([line.startswith('{') and line.endswith('}')
                    for line in e.stdout.splitlines()]):
                # we have the usual json output on stdout. Therefore we can
                # probably return and don't need to raise; so get stdout
                # for json loading:
                out = e.stdout
            else:
                out = None

            # Note: Workaround for not existing files as long as annex doesn't
            # report it within JSON response:
            # see http://git-annex.branchable.com/bugs/copy_does_not_reflect_some_failed_copies_in_--json_output/
            not_existing = [
                line.split()[1] for line in e.stderr.splitlines()
                if line.startswith('git-annex:') and
                   line.endswith('not found')
            ]
            if not_existing:
                if out is None:
                    # we create the error reporting herein. If all files were
                    # not found, there is nothing on stdout and we don't need
                    # anything
                    out = ""
                if not out.endswith(linesep):
                    out += linesep
                out += linesep.join(
                    ['{{"command": "{cmd}", "file": "{path}", '
                     '"note": "{note}",'
                     '"success":false}}'.format(
                         cmd=command, path=f, note="not found")
                     for f in not_existing])

            # Note: insert additional code here to analyse failure and possibly
            # raise a custom exception

            # if we didn't raise before, just depend on whether or not we seem
            # to have some json to return. It should contain information on
            # failure in keys 'success' and 'note'
            # TODO: This is not entirely true. 'annex status' may return empty,
            # while there was a 'fatal:...' in stderr, which should be a
            # failure/exception
            # Or if we had empty stdout but there was stderr
            if out is None or (not out and e.stderr):
                raise e
            if e.stderr:
                # else just warn about present errors
                shorten = lambda x: x[:1000] + '...' if len(x) > 1000 else x
                lgr.warning(
                    "Running %s resulted in stderr output: %s",
                    command, shorten(e.stderr)
                )
        finally:
            if progress_indicators:
                progress_indicators.finish()

        json_objects = (json_loads(line)
                        for line in out.splitlines() if line.startswith('{'))
        # protect against progress leakage
        json_objects = [j for j in json_objects if 'byte-progress' not in j]
        return json_objects

    # TODO: reconsider having any magic at all and maybe just return a list/dict always
    @normalize_paths
    def whereis(self, files, output='uuids', key=False, options=None, batch=False):
        """Lists repositories that have actual content of file(s).

        Parameters
        ----------
        files: list of str
            files to look for
        output: {'descriptions', 'uuids', 'full'}, optional
            If 'descriptions', a list of remotes descriptions returned is per
            each file. If 'full', for each file a dictionary of all fields
            is returned as returned by annex
        key: bool, optional
            Either provided files are actually annex keys
        options: list, optional
            Options to pass into git-annex call

        Returns
        -------
        list of list of unicode  or dict
            if output == 'descriptions', contains a list of descriptions of remotes
            for each input file, describing the remote for each remote, which
            was found by git-annex whereis, like::

                u'me@mycomputer:~/where/my/repo/is [origin]' or
                u'web' or
                u'me@mycomputer:~/some/other/clone'

            if output == 'uuids', returns a list of uuids.
            if output == 'full', returns a dictionary with filenames as keys
            and values a detailed record, e.g.::

                {'00000000-0000-0000-0000-000000000001': {
                  'description': 'web',
                  'here': False,
                  'urls': ['http://127.0.0.1:43442/about.txt', 'http://example.com/someurl']
                }}
        """
        if batch:
            lgr.warning("TODO: --batch mode for whereis.  Operating serially")

        OUTPUTS = {'descriptions', 'uuids', 'full'}
        if output not in OUTPUTS:
            raise ValueError(
                "Unknown value output=%r. Known are %s"
                % (output, ', '.join(map(repr, OUTPUTS)))
            )

        options = assure_list(options, copy=True)
        if key:
            kwargs = {'opts': options + ["--key"] + files}
        else:
            kwargs = {'files': files}

        json_objects = self._run_annex_command_json('whereis', **kwargs)
        if output in {'descriptions', 'uuids'}:
            return [
                [remote.get(output[:-1]) for remote in j.get('whereis')]
                if j.get('success') else []
                for j in json_objects
            ]
        elif output == 'full':
            # TODO: we might want to optimize storage since many remotes entries will be the
            # same so we could just reuse them instead of brewing copies
            return {
                j['key' if (key or '--all' in options) else 'file']
                : self._whereis_json_to_dict(j)
                for j in json_objects
                if not j.get('key').endswith('.this-is-a-test-key')
            }

    # TODO:
    # I think we should make interface cleaner and less ambigious for those annex
    # commands which could operate on globs, files, and entire repositories, separating
    # those out, e.g. annex_info_repo, annex_info_files at least.
    # If we make our calling wrappers work without relying on invoking from repo topdir,
    # then returned filenames would not need to be mapped, so we could easily work on dirs
    # and globs.
    # OR if explicit filenames list - return list of matching entries, if globs/dirs -- return dict?
    @normalize_paths(map_filenames_back=True)
    def info(self, files, batch=False, fast=False):
        """Provide annex info for file(s).

        Parameters
        ----------
        files: list of str
            files to look for

        Returns
        -------
        dict
          Info for each file
        """

        options = ['--bytes', '--fast'] if fast else ['--bytes']

        if not batch:
            json_objects = self._run_annex_command_json(
                'info', opts=options, files=files)
        else:
            json_objects = self._batched.get(
                'info',
                annex_options=options, json=True, path=self.path
            )(files)

        # Some aggressive checks. ATM info can be requested only per file
        # json_objects is a generator, let's keep it that way
        # assert(len(json_objects) == len(files))
        # and that they all have 'file' equal to the passed one
        out = {}
        for j, f in zip(json_objects, files):
            assert(j.pop('file') == f)
            if not j['success']:
                j = None
            else:
                assert(j.pop('success') is True)
                # convert size to int
                j['size'] = int(j['size']) if 'unknown' not in j['size'] else None
                # and pop the "command" field
                j.pop("command")
            out[f] = j
        return out

    def repo_info(self, fast=False):
        """Provide annex info for the entire repository.

        Returns
        -------
        dict
          Info for the repository, with keys matching the ones returned by annex
        """

        options = ['--bytes', '--fast'] if fast else ['--bytes']

        json_records = list(self._run_annex_command_json('info', opts=options))
        assert(len(json_records) == 1)

        # TODO: we need to abstract/centralize conversion from annex fields
        # For now just tune up few for immediate usability
        info = json_records[0]
        for k in info:
            if k.endswith(' size') or k.endswith(' disk space') or k.startswith('size of '):
                size = info[k].split()[0]
                if size.isdigit():
                    info[k] = int(size)
                else:
                    lgr.debug("Size %r reported to be %s, setting to None", k, size)
                    info[k] = None
        assert(info.pop('success'))
        assert(info.pop('command') == 'info')
        return info  # just as is for now

    def get_annexed_files(self, with_content_only=False, patterns=None):
        """Get a list of files in annex

        Parameters
        ----------
        with_content_only : bool, optional
            Only list files whose content is present.
        patterns : list, optional
            Globs to pass to annex's `--include=`. Files that match any of
            these will be returned (i.e., they'll be separated by `--or`).

        Returns
        -------
        A list of file names
        """
        if not patterns:
            args = [] if with_content_only else ['--include', "*"]
        else:
            if len(patterns) == 1:
                args = ['--include', patterns[0]]
            else:
                args = ['-(']
                for pat in patterns[:-1]:
                    args.extend(['--include', pat, "--or"])
                args.extend(['--include', patterns[-1]])
                args.append('-)')

            if with_content_only:
                args.extend(['--in', 'here'])
        out, err = self._run_annex_command('find', annex_options=args)
        # TODO: JSON
        return out.splitlines()

    def get_preferred_content(self, property, remote=None):
        """Get preferred content configuration of a repository or remote

        Parameters
        ----------
        property : {'wanted', 'required', 'group'}
          Type of property to query
        remote : str, optional
          If not specified (None), returns the property for the local
          repository.

        Returns
        -------
        str
          Either the setting is returned, or an empty string if there
          is none.

        Raises
        ------
        ValueError
          If an unknown property label is given.

        CommandError
          If the annex call errors.
        """
        if property not in ('wanted', 'required', 'group'):
            raise ValueError(
                'unknown preferred content property: {}'.format(property))
        return self._run_simple_annex_command(
            property,
            annex_options=[remote or '.'])

    def set_preferred_content(self, property, expr, remote=None):
        """Set preferred content configuration of a repository or remote

        Parameters
        ----------
        property : {'wanted', 'required', 'group'}
          Type of property to query
        expr : str
          Any expression or label supported by git-annex for the
          given property.
        remote : str, optional
          If not specified (None), sets the property for the local
          repository.

        Returns
        -------
        str
          Raw git-annex output in response to the set command.

        Raises
        ------
        ValueError
          If an unknown property label is given.

        CommandError
          If the annex call errors.
        """
        if property not in ('wanted', 'required', 'group'):
            raise ValueError(
                'unknown preferred content property: {}'.format(property))
        return self._run_simple_annex_command(
            property,
            annex_options=[remote or '.', expr])

    def get_groupwanted(self, name):
        """Get `groupwanted` expression for a group `name`

        Parameters
        ----------
        name : str
           Name of the groupwanted group
        """
        return self._run_simple_annex_command(
            'groupwanted', annex_options=[name]
        )

    def set_groupwanted(self, name, expr):
        """Set `expr` for the `name` groupwanted"""
        return self._run_simple_annex_command(
            'groupwanted', annex_options=[name, expr]
        )

    def precommit(self):
        """Perform pre-commit maintenance tasks, such as closing all batched annexes
        since they might still need to flush their changes into index
        """
        if self._batched is not None:
            self._batched.close()
        super(AnnexRepo, self).precommit()

    @borrowdoc(GitRepo)
    def commit(self, msg=None, options=None, _datalad_msg=False,
               careless=True, files=None, proxy=False):
        self.precommit()

        if files:
            files = assure_list(files)

        # Note: `proxy` is for explicitly enforcing the use of git-annex-proxy
        #       in direct mode. This is needed in very special cases, which
        #       might go away once we figured out a better way. In any case, it
        #       should turn into something that is automatically considered and
        #       not done by the caller of this method.

        if proxy:
            if not self.is_direct_mode():
                raise CommandNotAvailableError(
                    cmd="git-annex-proxy",
                    msg="git-annex-proxy is available in direct mode only")
            else:
                if _datalad_msg:
                    msg = self._get_prefixed_commit_msg(msg)
                if not msg:
                    if options:
                        if "--allow-empty-message" not in options:
                            options.append("--allow-empty-message")
                    else:
                        options = ["--allow-empty-message"]

                # committing explicitly given paths in direct mode via proxy
                # used to fail, because absolute paths are used. Using annex
                # proxy this leads to an error (path outside repository)
                if files:
                    if options is None:
                        options = []
                    for i in range(len(files)):
                        if isabs(files[i]):
                            options.append(normpath(relpath(files[i],
                                                            start=self.path)))
                        else:
                            options.append(files[i])
                try:
                    self.proxy(['git', 'commit'] + (['-m', msg] if msg else []) +
                               (options if options else []),
                               expect_stderr=True, expect_fail=True)
                except CommandError as e:
                    if 'nothing to commit' in e.stdout:
                        if careless:
                            lgr.debug("nothing to commit in {}. "
                                      "Ignored.".format(self))
                        else:
                            raise
                    elif 'no changes added to commit' in e.stdout or \
                            'nothing added to commit' in e.stdout:
                        if careless:
                            lgr.debug("no changes added to commit in {}. "
                                      "Ignored.".format(self))
                        else:
                            raise
                    elif "did not match any file(s) known to git." in e.stderr:
                        # TODO:
                        # Improve FileNotInXXXXError classes to better deal with
                        # multiple files; Also consider PathOutsideRepositoryError
                        raise FileNotInRepositoryError(cmd=e.cmd,
                                                       msg="File(s) unknown to git",
                                                       code=e.code,
                                                       filename=linesep.join(
                                                    [l for l in e.stderr.splitlines()
                                                     if l.startswith("pathspec")]))
                    else:
                        raise
        else:

            # Note: See the note on `proxy` parameter at the top of this method.
            #       Trying to automatically use git-annex-proxy, whenever we
            #       fail to commit the usual way via options to git in direct
            #       mode. In particular this can happen if sth was staged via
            #       git-annex-proxy, which is needed for --update option for
            #       example.

            if files:
                # Raise FileNotInRepositoryError if `files` aren't tracked.
                try:
                    super(AnnexRepo, self).commit(
                        "dryrun", options=["--dry-run", "--no-status"],
                        files=files)
                except CommandError as e:
                    if not (self.is_direct_mode() and
                            AnnexRepo._is_annex_work_tree_message(e.stderr)):
                        raise
                    # The git call may fail with
                    #   fatal: this operation must be run in a work tree
                    #   fatal: 'git status --porcelain=2' failed in submodule ...
                    # But that error message doesn't matter for the purpose of
                    # the "file is tracked" check. It won't be shown if there
                    # is a pathspec error.
                    lgr.debug("Ignoring commit --dry-run failure")
            try:
                alt_index_file = None

                direct_mode = self.is_direct_mode()
                # we might need to avoid explicit paths
                files_to_commit = None if direct_mode else files
                if direct_mode and files:
                    # In direct mode, if we commit file(s) they would get
                    # committed directly into git ignoring possibly being
                    # staged by annex.  So, if not all files are committed, and
                    # assuming that what is to be committed is staged (!could be
                    # wrong assumption), we need to prepare a custom index, and
                    # commit without specifying any paths
                    changed_files = {
                        staged: set(self.get_changed_files(staged=staged))
                        for staged in (True,) #  False}
                    }
                    files_set = {
                        _normalize_path(self.path, f) if isabs(f) else f
                        for f in files}
                    files_notstaged = files_set.difference(changed_files[True])
                    # Lazy .add('.') results in ALL (performance?) files listed
                    # here even if they are not changed at all and could be ignored
                    #files_changed_notstaged = files_notstaged.difference(changed_files[False])
                    # if files_changed_notstaged:
                    #     # To be safe(r), let's just blow for now!
                    #     # We could be smart(er) and do all the git or annex add
                    #     # but let's see if we run into those cases first
                    #     import pdb; pdb.set_trace()
                    #     raise RuntimeError(
                    #         "To commit files in direct mode, please first git "
                    #         "or git-annex add them first!  Files: %s"
                    #         % ', '.join(files_changed_notstaged)
                    #     )
                    # Files which were staged but not among files
                    staged_not_to_commit = changed_files[True].difference(files_set)
                    if staged_not_to_commit:
                        # Need an alternative index_file
                        with make_tempfile() as index_file:
                            alt_index_file = index_file
                            index_tree = self.repo.git.write_tree()
                            self.repo.git.read_tree(index_tree,
                                                    index_output=index_file)
                            # Reset the files we are not to be committed
                            self._git_custom_command(
                                list(staged_not_to_commit),
                                ['git', 'reset'],
                                index_file=alt_index_file)

                            super(AnnexRepo, self).commit(
                                msg, options,
                                _datalad_msg=_datalad_msg,
                                careless=careless,
                                index_file=alt_index_file)

                    # in any case we will not specify files explicitly
                    files_to_commit = None
                if not alt_index_file:
                    super(AnnexRepo, self).commit(msg, options,
                                                  _datalad_msg=_datalad_msg,
                                                  careless=careless,
                                                  files=files_to_commit)
            except CommandError as e:
                # TODO: just remove this???
                if self.is_direct_mode() and \
                    AnnexRepo._is_annex_work_tree_message(e.stderr):
                    lgr.debug("Commit failed. "
                              "Trying to commit via git-annex-proxy.")
                    self.commit(msg, options, _datalad_msg=_datalad_msg,
                                careless=careless, files=files, proxy=True)
                else:
                    raise
            finally:
                if alt_index_file and os.path.exists(alt_index_file):
                    unlink(alt_index_file)

    @normalize_paths(match_return_type=False)
    def remove(self, files, force=False, **kwargs):
        """Remove files from git/annex (works in direct mode as well)

        Parameters
        ----------
        files
        force: bool, optional
        """

        # TODO: parameter 'force' unnecessary => kwargs / to_options
        self.precommit()  # since might interfere

        return super(AnnexRepo, self).remove(files, force=force,
                                             normalize_paths=False,
                                             **kwargs)

    def get_contentlocation(self, key, batch=False):
        """Get location of the key content

        Normally under .git/annex objects in indirect mode and within file
        tree in direct mode.

        Unfortunately there is no (easy) way to discriminate situations
        when given key is simply incorrect (not known to annex) or its content
        not currently present -- in both cases annex just silently exits with -1


        Parameters
        ----------
        key: str
            key
        batch: bool, optional
            initiate or continue with a batched run of annex contentlocation

        Returns
        -------
        str
            path relative to the top directory of the repository. If no content
            is present, empty string is returned
        """

        if not batch:
            try:
                out, err = self._run_annex_command('contentlocation',
                                                   annex_options=[key],
                                                   expect_fail=True)
                return out.rstrip(linesep).splitlines()[0]
            except CommandError:
                return ''
        else:
            return self._batched.get('contentlocation', path=self.path)(key)

    @normalize_paths(serialize=True)
    def is_available(self, file_, remote=None, key=False, batch=False):
        """Check if file or key is available (from a remote)

        In case if key or remote is misspecified, it wouldn't fail but just keep
        returning False, although possibly also complaining out loud ;)

        Parameters
        ----------
        file_: str
            Filename or a key
        remote: str, optional
            Remote which to check.  If None, possibly multiple remotes are checked
            before positive result is reported
        key: bool, optional
            Either provided files are actually annex keys
        batch: bool, optional
            Initiate or continue with a batched run of annex checkpresentkey

        Returns
        -------
        bool
            with True indicating that file/key is available from (the) remote
        """

        if key:
            key_ = file_
        else:
            key_ = self.get_file_key(file_)  # ?, batch=batch

        annex_input = (key_,) if not remote else (key_, remote)

        if not batch:
            try:
                out, err = self._run_annex_command('checkpresentkey',
                                                   annex_options=list(annex_input),
                                                   expect_fail=True)
                assert(not out)
                return True
            except CommandError:
                return False
        else:
            annex_cmd = ["checkpresentkey"] + ([remote] if remote else [])
            out = self._batched.get(
                ':'.join(annex_cmd), annex_cmd,
                path=self.path)(key_)
            try:
                return {
                    # happens on travis in direct/heavy-debug mode, that process
                    # exits and closes stdout (upon unknown key) before we could
                    # read it, so we get None as the stdout.
                    # see https://github.com/datalad/datalad/issues/2330
                    # but it is associated with an unknown key, and for consistency
                    # we report False there too, as to ''
                    None: False,
                    '': False,  # when remote is misspecified ... stderr carries the msg
                    '0': False,
                    '1': True,
                }[out]
            except KeyError:
                raise ValueError(
                    "Received output %r from annex, whenever expect 0 or 1" % out
                )

    @normalize_paths(match_return_type=False)
    def _annex_custom_command(
            self, files, cmd_str, log_stdout=True, log_stderr=True,
            log_online=False, expect_stderr=False, cwd=None, env=None,
            shell=None, expect_fail=False):
        """Allows for calling arbitrary commands.

        Helper for developing purposes, i.e. to quickly implement git-annex
        commands for proof of concept.

        Parameters
        ----------
        files: list of files
        cmd_str: str
            arbitrary command str. `files` is appended to that string.

        Returns
        -------
        stdout, stderr
        """
        cmd = shlex.split(cmd_str + " " + " ".join(files), posix=not on_windows) \
            if isinstance(cmd_str, string_types) \
            else cmd_str + files

        if self.fake_dates_enabled:
            env = self.add_fake_dates(env)

        return self.cmd_call_wrapper.run(
            cmd,
            log_stderr=log_stderr, log_stdout=log_stdout, log_online=log_online,
            expect_stderr=expect_stderr,
            cwd=cwd, env=env, shell=shell, expect_fail=expect_fail)

    @normalize_paths
    def migrate_backend(self, files, backend=None):
        """Changes the backend used for `file`.

        The backend used for the key-value of `files`. Only files currently
        present are migrated.
        Note: There will be no notification if migrating fails due to the
        absence of a file's content!

        Parameters
        ----------
        files: list
            files to migrate.
        backend: str
            specify the backend to migrate to. If none is given, the
            default backend of this instance will be used.
        """

        if self.is_direct_mode():
            raise CommandNotAvailableError(
                'git-annex migrate',
                "Command 'migrate' is not available in direct mode.")
        self._run_annex_command('migrate',
                                annex_options=files,
                                backend=backend)

    @normalize_paths
    def get_file_backend(self, files):
        """Get the backend currently used for file(s).

        Parameters
        ----------
        files: list of str

        Returns
        -------
        list of str
            For each file in input list indicates the used backend by a str
            like "SHA256E" or "MD5".
        """

        return [self.get_file_key(f).split('-')[0] for f in files]

    @property
    def default_backends(self):
        self.config.reload()
        backends = self.config.get("annex.backends", default=None)
        if backends:
            return backends.split()
        else:
            return None

    def fsck(self):
        self._run_annex_command('fsck')

    # TODO: we probably need to override get_file_content, since it returns the
    # symlink's target instead of the actual content.

    # We need --auto and --fast having exposed  TODO
    @normalize_paths(match_return_type=False)  # get a list even in case of a single item
    def copy_to(self, files, remote, options=None, jobs=None):
        """Copy the actual content of `files` to `remote`

        Parameters
        ----------
        files: str or list of str
            path(s) to copy
        remote: str
            name of remote to copy `files` to

        Returns
        -------
        list of str
           files successfully copied
        """

        # find --in here --not --in remote
        # TODO: full support of annex copy options would lead to `files` being
        # optional. This means to check for whether files or certain options are
        # given and fail or just pass everything as is and try to figure out,
        # what was going on when catching CommandError

        if remote not in self.get_remotes():
            raise ValueError("Unknown remote '{0}'.".format(remote))

        options = options[:] if options else []

        # Note:
        # In case of single path, 'annex copy' will fail, if it cannot copy it.
        # With multiple files, annex will just skip the ones, it cannot deal
        # with. We'll do the same and report back what was successful
        # (see return value).
        # Therefore raise telling exceptions before even calling annex:
        if len(files) == 1:
            if not isdir(files[0]):
                self.get_file_key(files[0])

        # TODO: RF -- logic is duplicated with get() -- the only difference
        # is the verb (copy, copy) or (get, put) and remote ('here', remote)?
        if '--key' not in options:
            expected_copys, copy_files = self._get_expected_files(
                files, ['--in', 'here', '--not', '--in', remote])
        else:
            copy_files = files
            assert(len(files) == 1)
            expected_copys = {files[0]: AnnexRepo.get_size_from_key(files[0])}

        if not copy_files:
            lgr.debug("No files found needing copying.")
            return []

        if len(copy_files) != len(files):
            lgr.debug("Actually copying %d files", len(copy_files))

        annex_options = ['--to=%s' % remote]
        if options:
            annex_options.extend(shlex.split(options))

        # TODO: provide more meaningful message (possibly aggregating 'note'
        #  from annex failed ones
        results = self._run_annex_command_json(
            'copy',
            opts=annex_options,
            files=files,  # copy_files,
            jobs=jobs,
            expected_entries=expected_copys,
            progress=True
            #log_stdout=True, log_stderr=not log_online,
            #log_online=log_online, expect_stderr=True
        )
        results_list = list(results)
        # XXX this is the only logic different ATM from get
        # check if any transfer failed since then we should just raise an Exception
        # for now to guarantee consistent behavior with non--json output
        # see https://github.com/datalad/datalad/pull/1349#discussion_r103639456
        from operator import itemgetter
        failed_copies = [e['file'] for e in results_list if not e['success']]
        good_copies = [
            e['file'] for e in results_list
            if e['success'] and
               e.get('note', '').startswith('to ')  # transfer did happen
        ]
        if failed_copies:
            # TODO: RF for new fancy scheme of outputs reporting
            raise IncompleteResultsError(
                results=good_copies, failed=failed_copies,
                msg="Failed to copy %d file(s)" % len(failed_copies))
        return good_copies

    @property
    def uuid(self):
        """Annex UUID

        Returns
        -------
        str
          Returns a the annex UUID, if there is any, or `None` otherwise.
        """
        if not self._uuid:
            if not self.repo:
                return None
            self._uuid = self.config.get('annex.uuid', default=None)
        return self._uuid

    def get_description(self, uuid=None):
        """Get annex repository description

        Parameters
        ----------
        uuid : str, optional
          For which remote (based on uuid) to report description for

        Returns
        -------
        str or None
          None returned if not found
        """
        info = self.repo_info(fast=True)
        match = \
            (lambda x: x['here']) \
            if uuid is None \
            else (lambda x: x['uuid'] == uuid)

        matches = list(set(chain.from_iterable(
            [
                [r['description'] for r in remotes if match(r)]
                for k, remotes in iteritems(info)
                if k.endswith(' repositories')
            ]
        )))

        if len(matches) == 1:
            # single hit as it should
            return matches[0]
        elif len(matches) == 2:
            lgr.warning(
                "Found multiple hits while searching. Returning first among: %s",
                str(matches)
            )
            return matches[0]
        else:
            return None

    def set_remote_url(self, name, url, push=False):
        """Set the URL a remote is pointing to

        Sets the URL of the remote `name`. Requires the remote to already exist.

        Parameters
        ----------
        name: str
          name of the remote
        url: str
        push: bool
          if True, set the push URL, otherwise the fetch URL;
          if True, additionally set annexurl to `url`, to make sure annex uses
          it to talk to the remote, since access via fetch URL might be
          restricted.
        """

        if push:
            # if we are to set a push url, also set 'annexUrl' for this remote,
            # in order to make git-annex use it, when talking to the remote.
            # (see http://git-annex.branchable.com/bugs/annex_ignores_pushurl_and_uses_only_url_upon___34__copy_--to__34__/)
            var = 'remote.{0}.{1}'.format(name, 'annexurl')
            self.config.set(var, url, where='local', reload=True)
        super(AnnexRepo, self).set_remote_url(name, url, push)

    def get_metadata(self, files, timestamps=False):
        """Query git-annex file metadata

        Parameters
        ----------
        files : str or list(str)
          One or more paths for which metadata is to be queried.
        timestamps: bool, optional
          If True, the output contains a '<metadatakey>-lastchanged'
          key for every metadata item, reflecting the modification
          time, as well as a 'lastchanged' key with the most recent
          modification time of any metadata item.

        Returns
        -------
        generator
          One tuple per file (could be more items than input arguments
          when directories are given). First tuple item is the filename,
          second item is a dictionary with metadata key/value pairs. Note that annex
          metadata tags are stored under the key 'tag', which is a
          regular metadata item that can be manipulated like any other.
        """
        if not files:
            return
        files = assure_list(files)
        opts = ['--json']
        for res in self._run_annex_command_json(
                'metadata', opts=opts, files=files):
            yield (
                res['file'],
                res['fields'] if timestamps else \
                {k: v for k, v in res['fields'].items()
                 if not k.endswith('lastchanged')})

    def set_metadata(
            self, files, reset=None, add=None, init=None,
            remove=None, purge=None, recursive=False):
        """Manipulate git-annex file-metadata

        Parameters
        ----------
        files : str or list(str)
          One or more paths for which metadata is to be manipulated.
          The changes applied to each file item are uniform. However,
          the result may not be uniform across files, depending on the
          actual operation.
        reset : dict, optional
          Metadata items matching keys in the given dict are (re)set
          to the respective values.
        add : dict, optional
          The values of matching keys in the given dict appended to
          any possibly existing values. The metadata keys need not
          necessarily exist before.
        init : dict, optional
          Metadata items for the keys in the given dict are set
          to the respective values, if the key is not yet present
          in a file's metadata.
        remove : dict, optional
          Values in the given dict are removed from the metadata items
          matching the respective key, if they exist in a file's metadata.
          Non-existing values, or keys do not lead to failure.
        purge : list, optional
          Any metadata item with a key matching an entry in the given
          list is removed from the metadata.
        recursive : bool, optional
          If False, fail (with CommandError) when directory paths
          are given as `files`.

        Returns
        -------
        generator
          JSON obj per modified file
        """

        def _genspec(expr, d):
            return [expr.format(k, v) for k, vs in d.items() for v in assure_list(vs)]

        args = []
        spec = []
        for expr, d in (('{}={}', reset),
                        ('{}+={}', add),
                        ('{}?={}', init),
                        ('{}-={}', remove)):
            if d:
                spec.extend(_genspec(expr, d))
        # prefix all with '-s' and extend arg list
        args.extend(j for i in zip(['-s'] * len(spec), spec) for j in i)
        if purge:
            # and all '-r' args
            args.extend(j for i in zip(['-r'] * len(purge), purge)
                        for j in i)
        if not args:
            return

        if recursive:
            args.append('--force')

        # Make sure that batch add/addurl operations are closed so that we can
        # operate on files that were just added.
        self.precommit()

        for jsn in self._run_annex_command_json(
                'metadata',
                args,
                files=files):
            yield jsn

    @staticmethod
    def _is_annex_work_tree_message(out):
        return re.match(
            r'.*This operation must be run in a work tree.*'
            r'git status.*failed in submodule',
            out,
            re.MULTILINE | re.DOTALL | re.IGNORECASE)

# TODO: Why was this commented out?
# @auto_repr
class BatchedAnnexes(dict):
    """Class to contain the registry of active batch'ed instances of annex for
    a repository
    """
    def __init__(self, batch_size=0, git_options=None):
        self.batch_size = batch_size
        self.git_options = git_options or []
        super(BatchedAnnexes, self).__init__()

    def get(self, codename, annex_cmd=None, **kwargs):
        if annex_cmd is None:
            annex_cmd = codename

        git_options = self.git_options + kwargs.pop('git_options', [])
        if self.batch_size:
            git_options += ['-c', 'annex.queuesize=%d' % self.batch_size]

        # START RF/BF: extend codename to respect different options the process
        # is running with
        # TODO: Eventually there should be more RF'ing, since the actually used
        # codenames are partially reflecting this already. Any options used
        # therein should go away, since they are now automatically included.
        options = kwargs.copy()
        options['git_options'] = git_options
        options['annex_cmd'] = annex_cmd
        for key in options:
            codename += ':{0}:{1}'.format(key, options[key])
        # END RF/BF

        if codename not in self:
            # Create a new git-annex process we will keep around
            self[codename] = BatchedAnnex(annex_cmd,
                                          git_options=git_options,
                                          **kwargs)
        return self[codename]

    def clear(self):
        """Override just to make sure we don't rely on __del__ to close all
        the pipes"""
        self.close()
        super(BatchedAnnexes, self).clear()

    def close(self):
        """Close communication to all the batched annexes

        It does not remove them from the dictionary though
        """
        for p in self.values():
            p.close()

    def __del__(self):
        self.close()


def readline_rstripped(stdout):
    #return iter(stdout.readline, b'').next().rstrip()
    return stdout.readline().rstrip()


def readlines_until_ok_or_failed(stdout, maxlines=100):
    """Read stdout until line ends with ok or failed"""
    out = ''
    i = 0
    lgr.log(3, "Trying to receive from %s" % stdout)
    while not stdout.closed:
        i += 1
        if maxlines > 0 and i > maxlines:
            raise IOError("Expected no more than %d lines. So far received: %r" % (maxlines, out))
        lgr.log(2, "Expecting a line")
        line = stdout.readline()
        lgr.log(2, "Received line %r" % line)
        out += line
        if re.match(r'^.*\b(failed|ok)$', line.rstrip()):
            break
    return out.rstrip()


def readline_json(stdout):
    toload = stdout.readline().strip()
    return json_loads(toload) if toload else {}


@auto_repr
class BatchedAnnex(object):
    """Container for an annex process which would allow for persistent communication
    """

    def __init__(self, annex_cmd, git_options=None, annex_options=None, path=None,
                 json=False,
                 output_proc=None):
        if not isinstance(annex_cmd, list):
            annex_cmd = [annex_cmd]
        self.annex_cmd = annex_cmd
        self.git_options = git_options if git_options else []
        annex_options = annex_options if annex_options else []
        self.annex_options = annex_options + (['--json'] if json else [])
        self.path = path
        if output_proc is None:
            output_proc = readline_json if json else readline_rstripped
        self.output_proc = output_proc
        self._process = None
        self._stderr_out = None
        self._stderr_out_fname = None

    def _initialize(self):
        # TODO -- should get all those options about --debug and --backend which are used/composed
        # in AnnexRepo class
        lgr.debug("Initiating a new process for %s" % repr(self))
        cmd = ['git'] + self.git_options + \
              ['annex'] + self.annex_cmd + self.annex_options + ['--batch']  # , '--debug']
        lgr.log(5, "Command: %s" % cmd)
        # TODO: look into _run_annex_command  to support default options such as --debug
        #
        # according to the internet wisdom there is no easy way with subprocess
        # while avoid deadlocks etc.  We would need to start a thread/subprocess
        # to timeout etc
        # kwargs = dict(bufsize=1, universal_newlines=True) if PY3 else {}
        self._stderr_out, self._stderr_out_fname = tempfile.mkstemp()
        self._process = Popen(
            cmd, stdin=PIPE, stdout=PIPE,  stderr=self._stderr_out,
            env=GitRunner.get_git_environ_adjusted(),
            cwd=self.path,
            bufsize=1,
            universal_newlines=True  # **kwargs
        )

    def _check_process(self, restart=False):
        """Check if the process was terminated and restart if restart

        Returns
        -------
        bool
          True if process was alive.
        str
          stderr if any recorded if was terminated
        """
        process = self._process
        ret = True
        ret_stderr = None
        if process and process.poll():
            lgr.warning("Process %s was terminated with returncode %s" % (process, process.returncode))
            ret_stderr = self.close(return_stderr=True)
            ret = False
        if self._process is None and restart:
            lgr.warning("Restarting the process due to previous failure")
            self._initialize()
        return ret, ret_stderr

    def __call__(self, cmds):
        """

        Parameters
        ----------
        cmds : str or tuple or list of (str or tuple)

        Returns
        -------
        str or list
          Output received from annex.  list in case if cmds was a list
        """
        # TODO: add checks -- may be process died off and needs to be reinitiated
        if not self._process:
            self._initialize()

        input_multiple = isinstance(cmds, list)
        if not input_multiple:
            cmds = [cmds]

        output = []

        for entry in cmds:
            if not isinstance(entry, string_types):
                entry = ' '.join(entry)
            entry = entry + '\n'
            lgr.log(5, "Sending %r to batched annex %s" % (entry, self))
            # apparently communicate is just a one time show
            # stdout, stderr = self._process.communicate(entry)
            # according to the internet wisdom there is no easy way with subprocess
            self._check_process(restart=True)
            process = self._process  # _check_process might have restarted it
            process.stdin.write(assure_bytes(entry) if PY2 else entry)
            process.stdin.flush()
            lgr.log(5, "Done sending.")
            still_alive, stderr = self._check_process(restart=False)
            # TODO: we might want to handle still_alive, e.g. to allow for
            #       a number of restarts/resends, but it should be per command
            #       since for some we cannot just resend the same query. But if
            #       it is just a "get"er - we could resend it few times
            # We are expecting a single line output
            # TODO: timeouts etc
            stdout = assure_unicode(self.output_proc(process.stdout)) \
                if not process.stdout.closed else None
            if stderr:
                lgr.warning("Received output in stderr: %r", stderr)
            lgr.log(5, "Received output: %r" % stdout)
            output.append(stdout)

        return output if input_multiple else output[0]

    def __del__(self):
        self.close()

    def close(self, return_stderr=False):
        """Close communication and wait for process to terminate

        Returns
        -------
        str
          stderr output if return_stderr and stderr file was there.
          None otherwise
        """
        ret = None
        if self._stderr_out:
            # close possibly still open fd
            os.fdopen(self._stderr_out).close()
            self._stderr_out = None
        if self._process:
            process = self._process
            lgr.debug(
                "Closing stdin of %s and waiting process to finish", process)
            process.stdin.close()
            process.stdout.close()
            process.wait()
            self._process = None
            lgr.debug("Process %s has finished", process)
        if self._stderr_out_fname and os.path.exists(self._stderr_out_fname):
            if return_stderr:
                with open(self._stderr_out_fname, 'r') as f:
                    ret = f.read()
            # remove the file where we kept dumping stderr
            unlink(self._stderr_out_fname)
            self._stderr_out_fname = None
        return ret


def _get_size_from_perc_complete(count, perc):
    """A helper to get full size if know % and corresponding size"""
    portion = (float(perc) / 100.)
    return int(math.ceil(int(count) / portion)) \
        if portion else 0


class ProcessAnnexProgressIndicators(object):
    """'Filter' for annex --json output to react to progress indicators

    Instance of this beast should be passed into log_stdout option
    for git-annex commands runner
    """

    def __init__(self, expected=None):
        """

        Parameters
        ----------
        expected: dict, optional
           key -> size, expected entries (e.g. downloads)
        """
        # looking forward for multiple downloads at the same time
        self.pbars = {}
        self.total_pbar = None
        self.expected = expected
        self.only_one_expected = expected and len(self.expected) == 1
        self._failed = 0
        self._succeeded = 0
        self.start()

    def start(self):
        if self.expected:
            from datalad.ui import ui
            total = sum(filter(bool, self.expected.values()))
            if len(self.expected) > 1:
                self.total_pbar = ui.get_progressbar(
                    label="Total", total=total)
                self.total_pbar.start()

    def __getitem__(self, id_):
        if self.pbars == 1:
            # regardless what it is, it is the one!
            # will happen e.g. in case of add_url_to_file since filename
            # is not even reported in the status and final msg just
            # reports the key which is not known before
            return self.pbars.items()[0]
        return self.pbars[id_]

    def _update_pbar(self, pbar, new_value):
        """Updates pbar while also updating possibly total pbar"""
        old_value = getattr(pbar, '_old_value', 0)
        # due to http://git-annex.branchable.com/bugs/__34__byte-progress__34___could_jump_down_upon_initiating_re-download_--_report_actual_one_first__63__/?updated
        # we will just skip the first update to avoid possible incorrect
        # reporting
        if not getattr(pbar, '_first_skipped', False):
            setattr(pbar, '_first_skipped', True)
            lgr.log(1, "Skipped first update of pbar %s", pbar)
            return
        setattr(pbar, '_old_value', new_value)
        diff = new_value - old_value
        if diff < 0:
            # so above didn't help!
            # use warnings not lgr.warn since we apparently swallow stuff
            # upstairs!  Also it would take care about issuing it only once
            lgr.debug(
                "Got negative diff for progressbar. old_value=%r, new_value=%r",
                old_value, new_value)
            return
        if self.total_pbar:
            self.total_pbar.update(diff, increment=True)
        pbar.update(new_value)

    def _log_info(self, msg):
        """Helper to log a message, so we need to clear up the pbars first"""
        if self.total_pbar:
            self.total_pbar.clear()
        for pbar in self.pbars.values():
            pbar.clear()
        lgr.info(msg)

    def __call__(self, line):
        try:
            j = json.loads(line)
        except:
            # if we fail to parse, just return this precious thing for
            # possibly further processing
            return line
        # Process some messages which remotes etc might push to us
        if list(j) == ['info']:
            # Just INFO was received without anything else -- we log it at INFO
            info = j['info']
            if info.startswith('PROGRESS-JSON: '):
                j_ = json_loads(info[len('PROGRESS-JSON: '):])
                if ('command' in j_ and 'key' in j_) or 'byte-progress' in j_:
                    j = j_
                else:
                    self._log_info(info)
            else:
                self._log_info(info)
                return

        target_size = None
        if 'command' in j and ('key' in j or 'file' in j):
            # might be the finish line message
            action_item = j.get('key') or j.get('file')
            j_action_id = (j['command'], action_item)
            pbar = self.pbars.pop(j_action_id, None)
            if pbar is None and len(self.pbars) == 1 and self.only_one_expected:
                # it is the only one left - take it!
                pbar = self.pbars.popitem()[1]
            if j.get('success') in {True, 'true'}:
                self._succeeded += 1
                if pbar:
                    self._update_pbar(pbar, pbar.total)
                elif self.total_pbar:
                    # we didn't have a pbar for this download, so total should
                    # get it all at once
                    try:
                        target_size = self.expected[action_item]
                    except:
                        target_size = None
                    if not target_size and 'key' in j:
                        target_size = AnnexRepo.get_size_from_key(j['key'])
                    self.total_pbar.update(target_size, increment=True)
            else:
                lgr.log(5, "Message with failed status: %s" % str(j))
                self._failed += 1

            if self.total_pbar:
                failed_str = (
                    ", " + ansi_colors.color_word("%d failed" % self._failed,
                                                  ansi_colors.RED)) \
                    if self._failed else ''

                self.total_pbar.set_desc(
                    "Total (%d ok%s out of %d)" % (
                        self._succeeded,
                        failed_str,
                        len(self.expected)
                        if self.expected
                        else self._succeeded + self._failed))
                # seems to be of no effect to force it repaint
                self.total_pbar.refresh()

            if pbar:
                pbar.finish()

        if 'byte-progress' not in j:
            # some other thing than progress
            return line

        # so we have a progress indicator, let's deal with it
        action = j['action']
        action_item = action.get('key') or action.get('file')
        action_id = (action['command'], action_item)
        if action_id not in self.pbars:
            # New download!
            from datalad.ui import ui
            from datalad.ui import utils as ui_utils
            # TODO: whenever target size gets reported -- used it!
            # http://git-annex.branchable.com/todo/interface_to_the___34__progress__34___of_annex_operations/#comment-6bbc26aae9867603863050ddcb09a9a0
            # for now deduce from key or approx from '%'
            # TODO: unittest etc to check when we have a relaxed
            # URL without any size known in advance
            if not target_size:
                target_size = \
                    AnnexRepo.get_size_from_key(action.get('key')) or \
                    _get_size_from_perc_complete(
                        j['byte-progress'],
                        j.get('percent-progress', '').rstrip('%')
                    ) or \
                    0
            w = ui_utils.get_console_width()

            if not action_item and self.only_one_expected:
                # must be the one!
                action_item = list(self.expected)[0]

            title = str(action.get('file') or action_item)

            pbar_right = 50
            title_len = w - pbar_right - 4  # (4 for reserve)
            if len(title) > title_len:
                half = title_len//2 - 2
                title = '%s .. %s' % (title[:half], title[-half:])
            pbar = self.pbars[action_id] = ui.get_progressbar(
                label=title, total=target_size)
            pbar.start()

        lgr.log(1, "Updating pbar for action_id=%s. annex: %s.\n",
                action_id, j)
        self._update_pbar(
            self[action_id],
            int(j.get('byte-progress'))
        )

    def finish(self):
        if self.total_pbar:
            self.total_pbar.finish()
            self.total_pbar = None
        if self.pbars:
            lgr.warning("Still have %d active progress bars when stopping",
                        len(self.pbars))
        for pbar in self.pbars.values():
            pbar.finish()
        self.pbars = {}
        self._failed = 0
        self._succeeded = 0

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user