{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface for dataset (component) installation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nimport os\nfrom os.path import join as opj, abspath, relpath, pardir, isabs, isdir, \\\n    exists, islink, sep, realpath\n\nfrom datalad.cmd import CommandError\nfrom datalad.cmd import Runner\nfrom datalad.distribution.dataset import Dataset, datasetmethod, \\\n    resolve_path, EnsureDataset\nfrom datalad.interface.base import Interface\nfrom datalad.support.annexrepo import AnnexRepo, FileInGitError, \\\n    FileNotInAnnexError\nfrom datalad.support.constraints import EnsureStr, EnsureNone, EnsureChoice, \\\n    EnsureBool\nfrom datalad.support.exceptions import InsufficientArgumentsError\nfrom datalad.support.gitrepo import GitRepo, GitCommandError\nfrom datalad.support.param import Parameter\nfrom datalad.utils import expandpath, knows_annex, assure_dir, \\\n    is_explicit_path, on_windows, swallow_logs\nfrom datalad.support.network import get_local_path_from_url\nfrom datalad.support.network import is_url\nfrom datalad.utils import rmtree\n\nlgr = logging.getLogger('datalad.distribution.install')\n\n\ndef _with_sep(path):\n    \"\"\"Little helper to guarantee that path ends with /\"\"\"\n    return path + sep if not path.endswith(sep) else path\n\n\ndef _installationpath_from_url(url):\n    \"\"\"Returns a relative path derived from the trailing end of a URL\n\n    This can be used to determine an installation path of a Dataset\n    from a URL, analog to what `git clone` does.\n    \"\"\"\n    path = url.rstrip('/')\n    if '/' in path:\n        path = path.split('/')\n        if path[-1] == '.git':\n            path = path[-2]\n        else:\n            path = path[-1]\n    if path.endswith('.git'):\n        path = path[:-4]\n    return path\n\n\ndef get_git_dir(path):\n    \"\"\"figure out a repo's gitdir\n\n    '.git' might be a  directory, a symlink or a file\n\n    Parameter\n    ---------\n    path: str\n      currently expected to be the repos base dir\n\n    Returns\n    -------\n    str\n      relative path to the repo's git dir; So, default would be \".git\"\n    \"\"\"\n\n    from os.path import isfile\n    from os import readlink\n\n    dot_git = opj(path, \".git\")\n    if not exists(dot_git):\n        raise RuntimeError(\"Missing .git in %s.\" % path)\n    elif islink(dot_git):\n        git_dir = readlink(dot_git)\n    elif isdir(dot_git):\n        git_dir = \".git\"\n    elif isfile(dot_git):\n        with open(dot_git) as f:\n            git_dir = f.readline()\n            if git_dir.startswith(\"gitdir:\"):\n                git_dir = git_dir[7:]\n            git_dir = git_dir.strip()\n\n    return git_dir\n\n\ndef _install_subds_from_flexible_source(ds, sm_path, sm_url, recursive):\n    \"\"\"Tries to obtain a given subdataset from several meaningful locations\"\"\"\n    # shortcut\n    vcs = ds.repo\n    repo = vcs.repo\n    # compose a list of candidate clone URLs\n    clone_urls = []\n    # if we have a remote, let's check the location of that remote\n    # for the presence of the desired submodule\n    tracking_branch = repo.active_branch.tracking_branch()\n    remote_url = ''\n    # remember suffix\n    url_suffix = ''\n    if tracking_branch:\n        # name of the default remote for the active branch\n        remote_name = repo.active_branch.tracking_branch().remote_name\n        remote_url = vcs.get_remote_url(remote_name, push=False)\n        if remote_url.rstrip('/').endswith('/.git'):\n            url_suffix = '/.git'\n            remote_url = remote_url[:-5]\n        # attempt: submodule checkout at parent remote URL\n        clone_urls.append('{0}/{1}{2}'.format(\n            remote_url, sm_path, url_suffix))\n    # attempt: configured submodule URL\n    if sm_url.startswith('/') or is_url(sm_url):\n        # this seems to be an absolute location -> take as is\n        clone_urls.append(sm_url)\n    else:\n        # need to resolve relative URL\n        if not remote_url:\n            # we have no remote URL, hence we need to go with the\n            # local path\n            remote_url = ds.path\n        remote_url_l = remote_url.split('/')\n        sm_url_l = sm_url.split('/')\n        for i, c in enumerate(sm_url_l):\n            if c == pardir:\n                remote_url_l = remote_url_l[:-1]\n            else:\n                clone_urls.append('{0}/{1}{2}'.format(\n                    '/'.join(remote_url_l),\n                    '/'.join(sm_url_l[i:]),\n                    url_suffix))\n                break\n    # now loop over all candidates and try to clone\n    subds = Dataset(opj(ds.path, sm_path))\n    for clone_url in clone_urls:\n        lgr.debug(\"Attempt clone of subdataset from: {0}\".format(clone_url))\n        try:\n            subds = Install.__call__(\n                dataset=subds, path=None, source=clone_url,\n                recursive=recursive, add_data_to_git=False)\n        except GitCommandError:\n            # TODO: failed clone might leave something behind that causes the\n            # next attempt to fail as well. Implement safe way to remove clone\n            # attempt left-overs.\n            continue\n        lgr.debug(\"Update cloned subdataset {0} in parent\".format(subds))\n        if sm_path in ds.get_subdatasets(absolute=False, recursive=False):\n            ds.repo.update_submodule(sm_path, init=True)\n        else:\n            # submodule is brand-new and previously unknown\n            ds.repo.add_submodule(sm_path, url=clone_url)\n        _fixup_submodule_dotgit_setup(ds, sm_path)\n        return subds\n\n\ndef _install_subds_inplace(ds, path, relativepath):\n    \"\"\"Register an existing repository in the repo tree as a submodule\"\"\"\n    # FLOW GUIDE EXIT POINT\n    # this is an existing repo and must be in-place turned into\n    # a submodule of this dataset\n    ds.repo.add_submodule(relativepath, url=None)\n    _fixup_submodule_dotgit_setup(ds, relativepath)\n    # return newly added submodule as a dataset\n    return Dataset(path)\n\n\ndef _fixup_submodule_dotgit_setup(ds, relativepath):\n    \"\"\"Implementation of our current of .git in a subdataset\n\n    Each subdataset/module has its own .git directory where a standalone\n    repository would have it. No gitdir files, no symlinks.\n    \"\"\"\n    # move .git to superrepo's .git/modules, remove .git, create\n    # .git-file\n    path = opj(ds.path, relativepath)\n    src_dotgit = get_git_dir(path)\n\n    # at this point install always yields the desired result\n    # just make sure\n    assert(src_dotgit == '.git')\n\n\ndef get_containing_subdataset(ds, path):\n    \"\"\"Given a base dataset and a relative path get containing subdataset\n\n    Parameters\n    ----------\n    ds : Dataset\n      Reference or base dataset\n    path : str\n      Path relative to the reference dataset\n\n    Returns\n    -------\n    Dataset\n    \"\"\"\n\n    if is_explicit_path(path) and not path.startswith(ds.path):\n        raise ValueError(\"path {0} not in dataset.\".format(path))\n\n    for subds in ds.get_subdatasets():\n        common = os.path.commonprefix((_with_sep(subds), _with_sep(path)))\n        if common.endswith(sep) and common == _with_sep(subds):\n            return Dataset(path=opj(ds.path, common))\n    return ds\n\n\n# TODO: check whether the following is done already:\n# install of existing submodule; recursive call; source should not be None!\n\nclass Install(Interface):\n    \"\"\"Install a dataset component or entire dataset(s).\n\n    This command can make arbitrary content available in a dataset. This\n    includes the fulfillment of existing subdataset or file handles\n    in a dataset, as well as the addition of sudatasets and such handles for\n    content available locally or remotely.\n    \"\"\"\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"specify the dataset to perform the install operation on.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the current working directory and/or the `path` given\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        path=Parameter(\n            args=(\"path\",),\n            doc=\"\"\"path/name of the installation target.  If no `source` is\n            provided, and no `dataset` is given or detected, this is\n            interpreted as the source URL of a dataset and a destination\n            path will be derived from the URL similar to 'git clone'\"\"\",\n            nargs=\"*\",\n            constraints=EnsureStr() | EnsureNone()),\n        source=Parameter(\n            args=(\"-s\", \"--source\",),\n            doc=\"url or local path of the installation source\",\n            constraints=EnsureStr() | EnsureNone()),\n        # TODO this probably needs --with-data and --recursive as a plain boolean\n        recursive=Parameter(\n            args=(\"-r\", \"--recursive\"),\n            constraints=EnsureChoice('datasets', 'data') | EnsureBool(),\n            doc=\"\"\"if set, all content is installed recursively, including\n            content of any subdatasets\"\"\"),\n        add_data_to_git=Parameter(\n            args=(\"--add-data-to-git\",),\n            action='store_true',\n            doc=\"\"\"flag whether to add data directly to Git, instead of\n            tracking data identity only.  Usually this is not desired,\n            as it inflates dataset sizes and impacts flexibility of data\n            transport\"\"\"))\n\n    @staticmethod\n    @datasetmethod(name='install')\n    def __call__(dataset=None, path=None, source=None, recursive=False,\n                 add_data_to_git=False):\n        lgr.debug(\"Installation attempt started\")\n        # shortcut\n        ds = dataset\n\n        if ds is not None and not isinstance(ds, Dataset):\n            ds = Dataset(ds)\n\n        if isinstance(path, list):\n            if not len(path):\n                # normalize value to expected state when nothing was provided\n                path = None\n            elif len(path) == 1:\n                # we can simply continue with the function as called with a\n                # single argument\n                path = path[0]\n            else:\n                lgr.debug(\"Installation of multiple targets was requested: {0}\".format(path))\n                return [Install.__call__(\n                        dataset=ds,\n                        path=p,\n                        source=source,\n                        recursive=recursive) for p in path]\n\n        # resolve the target location against the provided dataset\n        if path is not None:\n            # make sure it is not a URL, `resolve_path` cannot handle that\n            if is_url(path):\n                try:\n                    path = get_local_path_from_url(path)\n                    path = resolve_path(path, ds)\n                except ValueError:\n                    # URL doesn't point to a local something\n                    pass\n            else:\n                path = resolve_path(path, ds)\n\n        # any `path` argument that point to something local now resolved and\n        # is no longer a URL\n\n        # if we have no dataset given, figure out which one we need to operate\n        # on, based on the resolved target location (that is now guaranteed to\n        # be specified, but only if path isn't a URL (anymore) -> special case,\n        # handles below\n        if ds is None and path is not None and not is_url(path):\n            # try to find a dataset at or above the installation target\n            dspath = GitRepo.get_toppath(abspath(path))\n            if dspath is None:\n                # no top-level dataset found, use path as such\n                dspath = path\n            ds = Dataset(dspath)\n\n        if ds is None and source is None and path is not None:\n            # no dataset, no source\n            # this could be a shortcut install call, where the first\n            # arg identifies the source\n            if is_url(path) or os.path.exists(path):\n                # we have an actual URL -> this should be the source\n                # OR\n                # it is not a URL, but it exists locally\n                lgr.debug(\n                    \"Single argument given to install and no dataset found. \"\n                    \"Assuming the argument identifies a source location.\")\n                source = path\n                path = None\n\n        lgr.debug(\"Resolved installation target: {0}\".format(path))\n\n        if ds is None and path is None and source is not None:\n            # we got nothing but a source. do something similar to git clone\n            # and derive the path from the source and continue\n            lgr.debug(\n                \"Neither dataset not target installation path provided. \"\n                \"Assuming installation of a remote dataset. \"\n                \"Deriving destination path from given source {0}\".format(\n                    source))\n            ds = Dataset(_installationpath_from_url(source))\n\n        if not path and ds is None:\n            # no dataset, no target location, nothing to do\n            raise InsufficientArgumentsError(\n                \"insufficient information for installation (needs at \"\n                \"least a dataset or an installation path\")\n\n        assert(ds is not None)\n\n        lgr.info(\"Installing {0}\".format(path if path else ds))\n\n        vcs = ds.repo\n        if vcs is None:\n            # TODO check that a \"ds.path\" actually points to a TOPDIR\n            # should be the case already, but maybe nevertheless check\n            existed = ds.path and exists(ds.path)\n\n            # We possibly need to consider /.git URL\n            candidate_sources = [source]\n            if source and not source.rstrip('/').endswith('/.git'):\n                candidate_sources.append('{0}/.git'.format(source.rstrip('/')))\n\n            for source_ in candidate_sources:\n                try:\n                    lgr.debug(\"Retrieving a dataset from URL: {0}\".format(source_))\n                    with swallow_logs():\n                        vcs = Install._get_new_vcs(ds, source_)\n                    break  # do not bother with other sources if succeeded\n                except GitCommandError:\n                    lgr.debug(\"Failed to retrieve from URL: {0}\".format(source_))\n                    if not existed and ds.path and exists(ds.path):\n                        lgr.debug(\"Wiping out unsuccessful clone attempt at {}\".format(ds.path))\n                        rmtree(ds.path)\n                    if source_ == candidate_sources[-1]:\n                        # Re-raise if failed even with the last candidate\n                        lgr.debug(\"Unable to establish repository instance at {0} from {1}\"\n                                  \"\".format(ds.path, candidate_sources))\n                        raise\n\n        assert(ds.repo)  # is automagically re-evaluated in the .repo property\n\n        runner = Runner()\n\n        if path is None or path == ds.path:\n            # if the goal was to install this dataset, we are done,\n            # except for 'recursive'.\n\n            # TODO: For now 'recursive' means just submodules.\n            # See --with-data vs. -- recursive and figure it out\n            if recursive:\n                for sm in ds.repo.get_submodules():\n                    _install_subds_from_flexible_source(\n                        ds, sm.path, sm.url, recursive=recursive)\n            return ds\n\n        # at this point this dataset is \"installed\", now we can test whether to\n        # install something into the dataset\n\n        # needed by the logic below\n        assert(isabs(path))\n\n        # express the destination path relative to the root of this dataset\n        relativepath = relpath(path, start=ds.path)\n        if path.startswith(pardir):\n            raise ValueError(\"installation path outside dataset\")\n\n        lgr.debug(\n            \"Resolved installation target relative to dataset {0}: {1}\".format(\n                ds, relativepath))\n\n        # this dataset must already know everything necessary\n        ###################################################\n        # FLOW GUIDE\n        #\n        # at this point we know nothing about the\n        # installation targether\n        ###################################################\n        try:\n            # it is simplest to let annex tell us what we are dealing with\n            lgr.debug(\"Trying to fetch file %s using annex\", relativepath)\n            if not isinstance(vcs, AnnexRepo):\n                assert(isinstance(vcs, GitRepo))\n                # FLOW GUIDE\n                # this is not an annex repo, but we raise exceptions\n                # to be able to treat them alike in the special case handling\n                # below\n                if not exists(path):\n                    raise IOError(\"path doesn't exist yet, might need special handling\")\n                elif relativepath in vcs.get_indexed_files():\n                    # relativepath is in git\n                    raise FileInGitError(\"We need to handle it as known to git\")\n                else:\n                    raise FileNotInAnnexError(\"We don't have yet annex repo here\")\n            if vcs.get_file_key(relativepath):\n                # FLOW GUIDE EXIT POINT\n                # this is an annex'ed file -> get it\n                # TODO implement `copy --from` using `source`\n                # TODO fail if `source` is something strange\n                vcs.get(relativepath)\n                # return the absolute path to the installed file\n                return path\n\n        except FileInGitError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - a  file already checked into Git\n            # - known submodule\n            ###################################################\n            lgr.log(5, \"FileInGitError logic\")\n            if source is not None:\n                raise FileInGitError(\"File %s is already in git. Specifying source (%s) makes no sense\"\n                                     % (path, source))\n            # file is checked into git directly -> nothing to do\n            # OR this is a submodule of this dataset\n            submodule = [sm for sm in ds.repo.get_submodules()\n                         if sm.path == relativepath]\n            if not len(submodule):\n                # FLOW GUIDE EXIT POINT\n                # this is a file in Git and no submodule, just return its path\n                lgr.debug(\"Don't act, data already present in Git\")\n                return path\n            elif len(submodule) > 1:\n                raise RuntimeError(\n                    \"more than one submodule registered at the same path?\")\n            submodule = submodule[0]\n\n            # FLOW GUIDE EXIT POINT\n            # we are dealing with a known submodule (i.e. `source`\n            # doesn't matter) -> check it out\n            lgr.debug(\"Install subdataset at: {0}\".format(submodule.path))\n            subds = _install_subds_from_flexible_source(\n                ds, submodule.path, submodule.url, recursive=recursive)\n            return subds\n\n        except FileNotInAnnexError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - content of a subdataset\n            # - an untracked file in this dataset\n            # - an entire untracked/unknown existing subdataset\n            ###################################################\n            lgr.log(5, \"FileNotInAnnexError logic\")\n            subds = get_containing_subdataset(ds, relativepath)\n            if ds.path != subds.path:\n                # FLOW GUIDE EXIT POINT\n                # target path belongs to a known subdataset, hand\n                # installation over to it\n                return subds.install(\n                    path=relpath(path, start=subds.path),\n                    source=source,\n                    recursive=recursive,\n                    add_data_to_git=add_data_to_git)\n\n            # FLOW GUIDE\n            # this must be an untracked/existing something, so either\n            # - a file\n            # - a directory\n            # - an entire repository\n            if exists(opj(path, '.git')):\n                # FLOW GUIDE EXIT POINT\n                # this is an existing repo and must be in-place turned into\n                # a submodule of this dataset\n                return _install_subds_inplace(\n                    ds, path, relativepath)\n\n            # FLOW GUIDE EXIT POINT\n            # - untracked file or directory in this dataset\n            if isdir(path) and not recursive:\n                # this is a directory and we want --recursive for it\n                raise ValueError(\n                    \"installation of a directory requires the `recursive` flag\")\n\n            # few sanity checks\n            if source and abspath(source) != path:\n                raise ValueError(\n                    \"installation target already exists, but `source` points to \"\n                    \"another location (target: '{0}', source: '{0}'\".format(\n                        source, path))\n\n            if not add_data_to_git and not (isinstance(vcs, AnnexRepo)):\n                raise RuntimeError(\n                    \"Trying to install file(s) into a dataset \"\n                    \"with a plain Git repository. First initialize annex, or \"\n                    \"provide override flag.\")\n\n            # switch `add` procedure between Git and Git-annex according to flag\n            if add_data_to_git:\n                vcs.add(relativepath, git=True)\n                added_files = resolve_path(relativepath, ds)\n            else:\n                # do a blunt `annex add`\n                added_files = vcs.add(relativepath)\n                # return just the paths of the installed components\n                if isinstance(added_files, list):\n                    added_files = [resolve_path(i['file'], ds) for i in added_files]\n                else:\n                    added_files = resolve_path(added_files['file'], ds)\n            if added_files:\n                return added_files\n            else:\n                return None\n\n        except IOError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # more complicated special cases -- `path` is either\n            # - a file/subdataset in a not yet initialized but known\n            #   submodule\n            # - an entire untracked/unknown existing subdataset\n            # - non-existing content that should be installed from `source`\n            ###################################################\n            lgr.log(5, \"IOError logic\")\n            # we can end up here in two cases ATM\n            if (exists(path) or islink(path)) or source is None:\n                # FLOW GUIDE\n                # - target exists but this dataset's VCS rejects it,\n                #   so it should be part of a subdataset\n                # or\n                # - target doesn't exist, but no source is given, so\n                #   it could be a handle that is actually contained in\n                #   a not yet installed subdataset\n                subds = get_containing_subdataset(ds, relativepath)\n                if ds.path != subds.path:\n                    # FLOW GUIDE\n                    # target path belongs to a subdataset, hand installation\n                    # over to it\n                    if not subds.is_installed():\n                        # FLOW GUIDE\n                        # we are dealing with a target in a not yet\n                        # available but known subdataset -> install it first\n                        ds.install(subds.path, recursive=recursive)\n                    return subds.install(\n                        path=relpath(path, start=subds.path),\n                        source=source,\n                        recursive=recursive,\n                        add_data_to_git=add_data_to_git)\n\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            if not source:\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            source_path = expandpath(source)\n            if exists(source_path):\n                # FLOW GUIDE EXIT POINT\n                # this could be\n                # - local file\n                # - local directory\n                # - repository outside the dataset\n                # we only want to support the last case of locally cloning\n                # a repo -- fail otherwise\n                if exists(opj(source_path, '.git')):\n                    return _install_subds_from_flexible_source(\n                        ds, relativepath, source_path, recursive)\n\n                raise ValueError(\n                    \"installing individual local files or directories is not \"\n                    \"supported, copy/move them into the dataset first\")\n\n            # FLOW GUIDE\n            # `source` is non-local, it could be:\n            #   - repository\n            #   - file\n            # we have no further evidence, hence we need to try\n            try:\n                # FLOW GUIDE EXIT POINT\n                # assume it is a dataset\n                return _install_subds_from_flexible_source(\n                    ds, relativepath, source, recursive)\n            except CommandError:\n                # FLOW GUIDE EXIT POINT\n                # apaarently not a repo, assume it is a file url\n                vcs.add_url_to_file(relativepath, source)\n                return path\n\n    @staticmethod\n    def _get_new_vcs(ds, source):\n        if source is None:\n            raise RuntimeError(\n                \"No `source` was provided. To create a new dataset \"\n                \"use the `create` command.\")\n        else:\n            # when obtained from remote, try with plain Git\n            lgr.info(\"Creating a new git repo at %s\", ds.path)\n            vcs = GitRepo(ds.path, url=source, create=True)\n            if knows_annex(ds.path):\n                # init annex when traces of a remote annex can be detected\n                lgr.info(\"Initializing annex repo at %s\", ds.path)\n                vcs = AnnexRepo(ds.path, init=True)\n            else:\n                lgr.debug(\"New repository clone has no traces of an annex\")\n        return vcs\n\n    @staticmethod\n    def result_renderer_cmdline(res):\n        from datalad.ui import ui\n        if res is None:\n            res = []\n        if not isinstance(res, list):\n            res = [res]\n        if not len(res):\n            ui.message(\"Nothing was installed\")\n            return\n        items = '\\n'.join(map(str, res))\n        msg = \"{n} installed {obj} available at\\n{items}\".format(\n            obj='items are' if len(res) > 1 else 'item is',\n            n=len(res),\n            items=items)\n        ui.message(msg)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface for dataset (component) installation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport logging\nimport os\nfrom os.path import join as opj, abspath, relpath, pardir, isabs, isdir, \\\n    exists, islink, sep, realpath\n\nfrom datalad.cmd import CommandError\nfrom datalad.cmd import Runner\nfrom datalad.distribution.dataset import Dataset, datasetmethod, \\\n    resolve_path, EnsureDataset\nfrom datalad.interface.base import Interface\nfrom datalad.support.annexrepo import AnnexRepo, FileInGitError, \\\n    FileNotInAnnexError\nfrom datalad.support.constraints import EnsureStr, EnsureNone, EnsureChoice, \\\n    EnsureBool\nfrom datalad.support.exceptions import InsufficientArgumentsError\nfrom datalad.support.gitrepo import GitRepo, GitCommandError\nfrom datalad.support.param import Parameter\nfrom datalad.utils import expandpath, knows_annex, assure_dir, \\\n    is_explicit_path, on_windows, swallow_logs\nfrom datalad.support.network import get_local_path_from_url\nfrom datalad.support.network import is_url\nfrom datalad.utils import rmtree\n\nlgr = logging.getLogger('datalad.distribution.install')\n\n\ndef _with_sep(path):\n    \"\"\"Little helper to guarantee that path ends with /\"\"\"\n    return path + sep if not path.endswith(sep) else path\n\n\ndef _installationpath_from_url(url):\n    \"\"\"Returns a relative path derived from the trailing end of a URL\n\n    This can be used to determine an installation path of a Dataset\n    from a URL, analog to what `git clone` does.\n    \"\"\"\n    path = url.rstrip('/')\n    if '/' in path:\n        path = path.split('/')\n        if path[-1] == '.git':\n            path = path[-2]\n        else:\n            path = path[-1]\n    if path.endswith('.git'):\n        path = path[:-4]\n    return path\n\n\ndef get_git_dir(path):\n    \"\"\"figure out a repo's gitdir\n\n    '.git' might be a  directory, a symlink or a file\n\n    Parameter\n    ---------\n    path: str\n      currently expected to be the repos base dir\n\n    Returns\n    -------\n    str\n      relative path to the repo's git dir; So, default would be \".git\"\n    \"\"\"\n\n    from os.path import isfile\n    from os import readlink\n\n    dot_git = opj(path, \".git\")\n    if not exists(dot_git):\n        raise RuntimeError(\"Missing .git in %s.\" % path)\n    elif islink(dot_git):\n        git_dir = readlink(dot_git)\n    elif isdir(dot_git):\n        git_dir = \".git\"\n    elif isfile(dot_git):\n        with open(dot_git) as f:\n            git_dir = f.readline()\n            if git_dir.startswith(\"gitdir:\"):\n                git_dir = git_dir[7:]\n            git_dir = git_dir.strip()\n\n    return git_dir\n\n\ndef _install_subds_from_flexible_source(ds, sm_path, sm_url, recursive):\n    \"\"\"Tries to obtain a given subdataset from several meaningful locations\"\"\"\n    # shortcut\n    vcs = ds.repo\n    repo = vcs.repo\n    # compose a list of candidate clone URLs\n    clone_urls = []\n    # if we have a remote, let's check the location of that remote\n    # for the presence of the desired submodule\n    tracking_branch = repo.active_branch.tracking_branch()\n    remote_url = ''\n    # remember suffix\n    url_suffix = ''\n    if tracking_branch:\n        # name of the default remote for the active branch\n        remote_name = repo.active_branch.tracking_branch().remote_name\n        remote_url = vcs.get_remote_url(remote_name, push=False)\n        if remote_url.rstrip('/').endswith('/.git'):\n            url_suffix = '/.git'\n            remote_url = remote_url[:-5]\n        # attempt: submodule checkout at parent remote URL\n        clone_urls.append('{0}/{1}{2}'.format(\n            remote_url, sm_path, url_suffix))\n    # attempt: configured submodule URL\n    if sm_url.startswith('/') or is_url(sm_url):\n        # this seems to be an absolute location -> take as is\n        clone_urls.append(sm_url)\n    else:\n        # need to resolve relative URL\n        if not remote_url:\n            # we have no remote URL, hence we need to go with the\n            # local path\n            remote_url = ds.path\n        remote_url_l = remote_url.split('/')\n        sm_url_l = sm_url.split('/')\n        for i, c in enumerate(sm_url_l):\n            if c == pardir:\n                remote_url_l = remote_url_l[:-1]\n            else:\n                clone_urls.append('{0}/{1}{2}'.format(\n                    '/'.join(remote_url_l),\n                    '/'.join(sm_url_l[i:]),\n                    url_suffix))\n                break\n    # now loop over all candidates and try to clone\n    subds = Dataset(opj(ds.path, sm_path))\n    for clone_url in clone_urls:\n        lgr.debug(\"Attempt clone of subdataset from: {0}\".format(clone_url))\n        try:\n            subds = Install.__call__(\n                dataset=subds, path=None, source=clone_url,\n                recursive=recursive, add_data_to_git=False)\n        except GitCommandError:\n            # TODO: failed clone might leave something behind that causes the\n            # next attempt to fail as well. Implement safe way to remove clone\n            # attempt left-overs.\n            continue\n        lgr.debug(\"Update cloned subdataset {0} in parent\".format(subds))\n        if sm_path in ds.get_subdatasets(absolute=False, recursive=False):\n            ds.repo.update_submodule(sm_path, init=True)\n        else:\n            # submodule is brand-new and previously unknown\n            ds.repo.add_submodule(sm_path, url=clone_url)\n        _fixup_submodule_dotgit_setup(ds, sm_path)\n        return subds\n\n\ndef _install_subds_inplace(ds, path, relativepath):\n    \"\"\"Register an existing repository in the repo tree as a submodule\"\"\"\n    # FLOW GUIDE EXIT POINT\n    # this is an existing repo and must be in-place turned into\n    # a submodule of this dataset\n    ds.repo.add_submodule(relativepath, url=None)\n    _fixup_submodule_dotgit_setup(ds, relativepath)\n    # return newly added submodule as a dataset\n    return Dataset(path)\n\n\ndef _fixup_submodule_dotgit_setup(ds, relativepath):\n    \"\"\"Implementation of our current of .git in a subdataset\n\n    Each subdataset/module has its own .git directory where a standalone\n    repository would have it. No gitdir files, no symlinks.\n    \"\"\"\n    # move .git to superrepo's .git/modules, remove .git, create\n    # .git-file\n    path = opj(ds.path, relativepath)\n    src_dotgit = get_git_dir(path)\n\n    # at this point install always yields the desired result\n    # just make sure\n    assert(src_dotgit == '.git')\n\n\ndef get_containing_subdataset(ds, path):\n    \"\"\"Given a base dataset and a relative path get containing subdataset\n\n    Parameters\n    ----------\n    ds : Dataset\n      Reference or base dataset\n    path : str\n      Path relative to the reference dataset\n\n    Returns\n    -------\n    Dataset\n    \"\"\"\n\n    if is_explicit_path(path) and not path.startswith(ds.path):\n        raise ValueError(\"path {0} not in dataset.\".format(path))\n\n    for subds in ds.get_subdatasets():\n        common = os.path.commonprefix((_with_sep(subds), _with_sep(path)))\n        if common.endswith(sep) and common == _with_sep(subds):\n            return Dataset(path=opj(ds.path, common))\n    return ds\n\n\n# TODO: check whether the following is done already:\n# install of existing submodule; recursive call; source should not be None!\n\nclass Install(Interface):\n    \"\"\"Install a dataset component or entire dataset(s).\n\n    This command can make arbitrary content available in a dataset. This\n    includes the fulfillment of existing :term:`subdataset` or file handles in\n    a dataset, as well as the addition of subdatasets and such handles for\n    content available locally or subdataset remotely.\n    \"\"\"\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"specify the dataset to perform the install operation on.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the current working directory and/or the `path` given\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        path=Parameter(\n            args=(\"path\",),\n            doc=\"\"\"path/name of the installation target.  If no `source` is\n            provided, and no `dataset` is given or detected, this is\n            interpreted as the source URL of a dataset and a destination\n            path will be derived from the URL similar to 'git clone'\"\"\",\n            nargs=\"*\",\n            constraints=EnsureStr() | EnsureNone()),\n        source=Parameter(\n            args=(\"-s\", \"--source\",),\n            doc=\"url or local path of the installation source\",\n            constraints=EnsureStr() | EnsureNone()),\n        # TODO this probably needs --with-data and --recursive as a plain boolean\n        recursive=Parameter(\n            args=(\"-r\", \"--recursive\"),\n            constraints=EnsureChoice('datasets', 'data') | EnsureBool(),\n            doc=\"\"\"if set, all content is installed recursively, including\n            content of any subdatasets\"\"\"),\n        add_data_to_git=Parameter(\n            args=(\"--add-data-to-git\",),\n            action='store_true',\n            doc=\"\"\"flag whether to add data directly to Git, instead of\n            tracking data identity only.  Usually this is not desired,\n            as it inflates dataset sizes and impacts flexibility of data\n            transport\"\"\"))\n\n    @staticmethod\n    @datasetmethod(name='install')\n    def __call__(dataset=None, path=None, source=None, recursive=False,\n                 add_data_to_git=False):\n        lgr.debug(\"Installation attempt started\")\n        # shortcut\n        ds = dataset\n\n        if ds is not None and not isinstance(ds, Dataset):\n            ds = Dataset(ds)\n\n        if isinstance(path, list):\n            if not len(path):\n                # normalize value to expected state when nothing was provided\n                path = None\n            elif len(path) == 1:\n                # we can simply continue with the function as called with a\n                # single argument\n                path = path[0]\n            else:\n                lgr.debug(\"Installation of multiple targets was requested: {0}\".format(path))\n                return [Install.__call__(\n                        dataset=ds,\n                        path=p,\n                        source=source,\n                        recursive=recursive) for p in path]\n\n        # resolve the target location against the provided dataset\n        if path is not None:\n            # make sure it is not a URL, `resolve_path` cannot handle that\n            if is_url(path):\n                try:\n                    path = get_local_path_from_url(path)\n                    path = resolve_path(path, ds)\n                except ValueError:\n                    # URL doesn't point to a local something\n                    pass\n            else:\n                path = resolve_path(path, ds)\n\n        # any `path` argument that point to something local now resolved and\n        # is no longer a URL\n\n        # if we have no dataset given, figure out which one we need to operate\n        # on, based on the resolved target location (that is now guaranteed to\n        # be specified, but only if path isn't a URL (anymore) -> special case,\n        # handles below\n        if ds is None and path is not None and not is_url(path):\n            # try to find a dataset at or above the installation target\n            dspath = GitRepo.get_toppath(abspath(path))\n            if dspath is None:\n                # no top-level dataset found, use path as such\n                dspath = path\n            ds = Dataset(dspath)\n\n        if ds is None and source is None and path is not None:\n            # no dataset, no source\n            # this could be a shortcut install call, where the first\n            # arg identifies the source\n            if is_url(path) or os.path.exists(path):\n                # we have an actual URL -> this should be the source\n                # OR\n                # it is not a URL, but it exists locally\n                lgr.debug(\n                    \"Single argument given to install and no dataset found. \"\n                    \"Assuming the argument identifies a source location.\")\n                source = path\n                path = None\n\n        lgr.debug(\"Resolved installation target: {0}\".format(path))\n\n        if ds is None and path is None and source is not None:\n            # we got nothing but a source. do something similar to git clone\n            # and derive the path from the source and continue\n            lgr.debug(\n                \"Neither dataset not target installation path provided. \"\n                \"Assuming installation of a remote dataset. \"\n                \"Deriving destination path from given source {0}\".format(\n                    source))\n            ds = Dataset(_installationpath_from_url(source))\n\n        if not path and ds is None:\n            # no dataset, no target location, nothing to do\n            raise InsufficientArgumentsError(\n                \"insufficient information for installation (needs at \"\n                \"least a dataset or an installation path\")\n\n        assert(ds is not None)\n\n        lgr.info(\"Installing {0}\".format(path if path else ds))\n\n        vcs = ds.repo\n        if vcs is None:\n            # TODO check that a \"ds.path\" actually points to a TOPDIR\n            # should be the case already, but maybe nevertheless check\n            existed = ds.path and exists(ds.path)\n\n            # We possibly need to consider /.git URL\n            candidate_sources = [source]\n            if source and not source.rstrip('/').endswith('/.git'):\n                candidate_sources.append('{0}/.git'.format(source.rstrip('/')))\n\n            for source_ in candidate_sources:\n                try:\n                    lgr.debug(\"Retrieving a dataset from URL: {0}\".format(source_))\n                    with swallow_logs():\n                        vcs = Install._get_new_vcs(ds, source_)\n                    break  # do not bother with other sources if succeeded\n                except GitCommandError:\n                    lgr.debug(\"Failed to retrieve from URL: {0}\".format(source_))\n                    if not existed and ds.path and exists(ds.path):\n                        lgr.debug(\"Wiping out unsuccessful clone attempt at {}\".format(ds.path))\n                        rmtree(ds.path)\n                    if source_ == candidate_sources[-1]:\n                        # Re-raise if failed even with the last candidate\n                        lgr.debug(\"Unable to establish repository instance at {0} from {1}\"\n                                  \"\".format(ds.path, candidate_sources))\n                        raise\n\n        assert(ds.repo)  # is automagically re-evaluated in the .repo property\n\n        runner = Runner()\n\n        if path is None or path == ds.path:\n            # if the goal was to install this dataset, we are done,\n            # except for 'recursive'.\n\n            # TODO: For now 'recursive' means just submodules.\n            # See --with-data vs. -- recursive and figure it out\n            if recursive:\n                for sm in ds.repo.get_submodules():\n                    _install_subds_from_flexible_source(\n                        ds, sm.path, sm.url, recursive=recursive)\n            return ds\n\n        # at this point this dataset is \"installed\", now we can test whether to\n        # install something into the dataset\n\n        # needed by the logic below\n        assert(isabs(path))\n\n        # express the destination path relative to the root of this dataset\n        relativepath = relpath(path, start=ds.path)\n        if path.startswith(pardir):\n            raise ValueError(\"installation path outside dataset\")\n\n        lgr.debug(\n            \"Resolved installation target relative to dataset {0}: {1}\".format(\n                ds, relativepath))\n\n        # this dataset must already know everything necessary\n        ###################################################\n        # FLOW GUIDE\n        #\n        # at this point we know nothing about the\n        # installation targether\n        ###################################################\n        try:\n            # it is simplest to let annex tell us what we are dealing with\n            lgr.debug(\"Trying to fetch file %s using annex\", relativepath)\n            if not isinstance(vcs, AnnexRepo):\n                assert(isinstance(vcs, GitRepo))\n                # FLOW GUIDE\n                # this is not an annex repo, but we raise exceptions\n                # to be able to treat them alike in the special case handling\n                # below\n                if not exists(path):\n                    raise IOError(\"path doesn't exist yet, might need special handling\")\n                elif relativepath in vcs.get_indexed_files():\n                    # relativepath is in git\n                    raise FileInGitError(\"We need to handle it as known to git\")\n                else:\n                    raise FileNotInAnnexError(\"We don't have yet annex repo here\")\n            if vcs.get_file_key(relativepath):\n                # FLOW GUIDE EXIT POINT\n                # this is an annex'ed file -> get it\n                # TODO implement `copy --from` using `source`\n                # TODO fail if `source` is something strange\n                vcs.get(relativepath)\n                # return the absolute path to the installed file\n                return path\n\n        except FileInGitError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - a  file already checked into Git\n            # - known submodule\n            ###################################################\n            lgr.log(5, \"FileInGitError logic\")\n            if source is not None:\n                raise FileInGitError(\"File %s is already in git. Specifying source (%s) makes no sense\"\n                                     % (path, source))\n            # file is checked into git directly -> nothing to do\n            # OR this is a submodule of this dataset\n            submodule = [sm for sm in ds.repo.get_submodules()\n                         if sm.path == relativepath]\n            if not len(submodule):\n                # FLOW GUIDE EXIT POINT\n                # this is a file in Git and no submodule, just return its path\n                lgr.debug(\"Don't act, data already present in Git\")\n                return path\n            elif len(submodule) > 1:\n                raise RuntimeError(\n                    \"more than one submodule registered at the same path?\")\n            submodule = submodule[0]\n\n            # FLOW GUIDE EXIT POINT\n            # we are dealing with a known submodule (i.e. `source`\n            # doesn't matter) -> check it out\n            lgr.debug(\"Install subdataset at: {0}\".format(submodule.path))\n            subds = _install_subds_from_flexible_source(\n                ds, submodule.path, submodule.url, recursive=recursive)\n            return subds\n\n        except FileNotInAnnexError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - content of a subdataset\n            # - an untracked file in this dataset\n            # - an entire untracked/unknown existing subdataset\n            ###################################################\n            lgr.log(5, \"FileNotInAnnexError logic\")\n            subds = get_containing_subdataset(ds, relativepath)\n            if ds.path != subds.path:\n                # FLOW GUIDE EXIT POINT\n                # target path belongs to a known subdataset, hand\n                # installation over to it\n                return subds.install(\n                    path=relpath(path, start=subds.path),\n                    source=source,\n                    recursive=recursive,\n                    add_data_to_git=add_data_to_git)\n\n            # FLOW GUIDE\n            # this must be an untracked/existing something, so either\n            # - a file\n            # - a directory\n            # - an entire repository\n            if exists(opj(path, '.git')):\n                # FLOW GUIDE EXIT POINT\n                # this is an existing repo and must be in-place turned into\n                # a submodule of this dataset\n                return _install_subds_inplace(\n                    ds, path, relativepath)\n\n            # FLOW GUIDE EXIT POINT\n            # - untracked file or directory in this dataset\n            if isdir(path) and not recursive:\n                # this is a directory and we want --recursive for it\n                raise ValueError(\n                    \"installation of a directory requires the `recursive` flag\")\n\n            # few sanity checks\n            if source and abspath(source) != path:\n                raise ValueError(\n                    \"installation target already exists, but `source` points to \"\n                    \"another location (target: '{0}', source: '{0}'\".format(\n                        source, path))\n\n            if not add_data_to_git and not (isinstance(vcs, AnnexRepo)):\n                raise RuntimeError(\n                    \"Trying to install file(s) into a dataset \"\n                    \"with a plain Git repository. First initialize annex, or \"\n                    \"provide override flag.\")\n\n            # switch `add` procedure between Git and Git-annex according to flag\n            if add_data_to_git:\n                vcs.add(relativepath, git=True)\n                added_files = resolve_path(relativepath, ds)\n            else:\n                # do a blunt `annex add`\n                added_files = vcs.add(relativepath)\n                # return just the paths of the installed components\n                if isinstance(added_files, list):\n                    added_files = [resolve_path(i['file'], ds) for i in added_files]\n                else:\n                    added_files = resolve_path(added_files['file'], ds)\n            if added_files:\n                return added_files\n            else:\n                return None\n\n        except IOError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # more complicated special cases -- `path` is either\n            # - a file/subdataset in a not yet initialized but known\n            #   submodule\n            # - an entire untracked/unknown existing subdataset\n            # - non-existing content that should be installed from `source`\n            ###################################################\n            lgr.log(5, \"IOError logic\")\n            # we can end up here in two cases ATM\n            if (exists(path) or islink(path)) or source is None:\n                # FLOW GUIDE\n                # - target exists but this dataset's VCS rejects it,\n                #   so it should be part of a subdataset\n                # or\n                # - target doesn't exist, but no source is given, so\n                #   it could be a handle that is actually contained in\n                #   a not yet installed subdataset\n                subds = get_containing_subdataset(ds, relativepath)\n                if ds.path != subds.path:\n                    # FLOW GUIDE\n                    # target path belongs to a subdataset, hand installation\n                    # over to it\n                    if not subds.is_installed():\n                        # FLOW GUIDE\n                        # we are dealing with a target in a not yet\n                        # available but known subdataset -> install it first\n                        ds.install(subds.path, recursive=recursive)\n                    return subds.install(\n                        path=relpath(path, start=subds.path),\n                        source=source,\n                        recursive=recursive,\n                        add_data_to_git=add_data_to_git)\n\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            if not source:\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            source_path = expandpath(source)\n            if exists(source_path):\n                # FLOW GUIDE EXIT POINT\n                # this could be\n                # - local file\n                # - local directory\n                # - repository outside the dataset\n                # we only want to support the last case of locally cloning\n                # a repo -- fail otherwise\n                if exists(opj(source_path, '.git')):\n                    return _install_subds_from_flexible_source(\n                        ds, relativepath, source_path, recursive)\n\n                raise ValueError(\n                    \"installing individual local files or directories is not \"\n                    \"supported, copy/move them into the dataset first\")\n\n            # FLOW GUIDE\n            # `source` is non-local, it could be:\n            #   - repository\n            #   - file\n            # we have no further evidence, hence we need to try\n            try:\n                # FLOW GUIDE EXIT POINT\n                # assume it is a dataset\n                return _install_subds_from_flexible_source(\n                    ds, relativepath, source, recursive)\n            except CommandError:\n                # FLOW GUIDE EXIT POINT\n                # apaarently not a repo, assume it is a file url\n                vcs.add_url_to_file(relativepath, source)\n                return path\n\n    @staticmethod\n    def _get_new_vcs(ds, source):\n        if source is None:\n            raise RuntimeError(\n                \"No `source` was provided. To create a new dataset \"\n                \"use the `create` command.\")\n        else:\n            # when obtained from remote, try with plain Git\n            lgr.info(\"Creating a new git repo at %s\", ds.path)\n            vcs = GitRepo(ds.path, url=source, create=True)\n            if knows_annex(ds.path):\n                # init annex when traces of a remote annex can be detected\n                lgr.info(\"Initializing annex repo at %s\", ds.path)\n                vcs = AnnexRepo(ds.path, init=True)\n            else:\n                lgr.debug(\"New repository clone has no traces of an annex\")\n        return vcs\n\n    @staticmethod\n    def result_renderer_cmdline(res):\n        from datalad.ui import ui\n        if res is None:\n            res = []\n        if not isinstance(res, list):\n            res = [res]\n        if not len(res):\n            ui.message(\"Nothing was installed\")\n            return\n        items = '\\n'.join(map(str, res))\n        msg = \"{n} installed {obj} available at\\n{items}\".format(\n            obj='items are' if len(res) > 1 else 'item is',\n            n=len(res),\n            items=items)\n        ui.message(msg)\n",
            "header": {
                "file": {
                    "path": "datalad/distribution/install.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "04559ac8c",
                "new_commit_hash": "816881f39",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/distribution/install.py"
            },
            "plus_file": {
                "path": "b/datalad/distribution/install.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 223,
                            "length": 9
                        },
                        "new_range": {
                            "start": 223,
                            "length": 9
                        },
                        "section": "class Install(Interface):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    \"\"\"Install a dataset component or entire dataset(s).",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    This command can make arbitrary content available in a dataset. This",
                                "type": "context"
                            },
                            {
                                "content": "    includes the fulfillment of existing subdataset or file handles",
                                "type": "deleted"
                            },
                            {
                                "content": "    in a dataset, as well as the addition of sudatasets and such handles for",
                                "type": "deleted"
                            },
                            {
                                "content": "    content available locally or remotely.",
                                "type": "deleted"
                            },
                            {
                                "content": "    includes the fulfillment of existing :term:`subdataset` or file handles in",
                                "type": "added"
                            },
                            {
                                "content": "    a dataset, as well as the addition of subdatasets and such handles for",
                                "type": "added"
                            },
                            {
                                "content": "    content available locally or subdataset remotely.",
                                "type": "added"
                            },
                            {
                                "content": "    \"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    _params_ = dict(",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/distribution/install.py",
                            "type": "class",
                            "name": "Install",
                            "content": "class Install(Interface):\n    \"\"\"Install a dataset component or entire dataset(s).\n\n    This command can make arbitrary content available in a dataset. This\n    includes the fulfillment of existing subdataset or file handles\n    in a dataset, as well as the addition of sudatasets and such handles for\n    content available locally or remotely.\n    \"\"\"\n\n    _params_ = dict(\n        dataset=Parameter(\n            args=(\"-d\", \"--dataset\"),\n            doc=\"\"\"specify the dataset to perform the install operation on.  If\n            no dataset is given, an attempt is made to identify the dataset\n            based on the current working directory and/or the `path` given\"\"\",\n            constraints=EnsureDataset() | EnsureNone()),\n        path=Parameter(\n            args=(\"path\",),\n            doc=\"\"\"path/name of the installation target.  If no `source` is\n            provided, and no `dataset` is given or detected, this is\n            interpreted as the source URL of a dataset and a destination\n            path will be derived from the URL similar to 'git clone'\"\"\",\n            nargs=\"*\",\n            constraints=EnsureStr() | EnsureNone()),\n        source=Parameter(\n            args=(\"-s\", \"--source\",),\n            doc=\"url or local path of the installation source\",\n            constraints=EnsureStr() | EnsureNone()),\n        # TODO this probably needs --with-data and --recursive as a plain boolean\n        recursive=Parameter(\n            args=(\"-r\", \"--recursive\"),\n            constraints=EnsureChoice('datasets', 'data') | EnsureBool(),\n            doc=\"\"\"if set, all content is installed recursively, including\n            content of any subdatasets\"\"\"),\n        add_data_to_git=Parameter(\n            args=(\"--add-data-to-git\",),\n            action='store_true',\n            doc=\"\"\"flag whether to add data directly to Git, instead of\n            tracking data identity only.  Usually this is not desired,\n            as it inflates dataset sizes and impacts flexibility of data\n            transport\"\"\"))\n\n    @staticmethod\n    @datasetmethod(name='install')\n    def __call__(dataset=None, path=None, source=None, recursive=False,\n                 add_data_to_git=False):\n        lgr.debug(\"Installation attempt started\")\n        # shortcut\n        ds = dataset\n\n        if ds is not None and not isinstance(ds, Dataset):\n            ds = Dataset(ds)\n\n        if isinstance(path, list):\n            if not len(path):\n                # normalize value to expected state when nothing was provided\n                path = None\n            elif len(path) == 1:\n                # we can simply continue with the function as called with a\n                # single argument\n                path = path[0]\n            else:\n                lgr.debug(\"Installation of multiple targets was requested: {0}\".format(path))\n                return [Install.__call__(\n                        dataset=ds,\n                        path=p,\n                        source=source,\n                        recursive=recursive) for p in path]\n\n        # resolve the target location against the provided dataset\n        if path is not None:\n            # make sure it is not a URL, `resolve_path` cannot handle that\n            if is_url(path):\n                try:\n                    path = get_local_path_from_url(path)\n                    path = resolve_path(path, ds)\n                except ValueError:\n                    # URL doesn't point to a local something\n                    pass\n            else:\n                path = resolve_path(path, ds)\n\n        # any `path` argument that point to something local now resolved and\n        # is no longer a URL\n\n        # if we have no dataset given, figure out which one we need to operate\n        # on, based on the resolved target location (that is now guaranteed to\n        # be specified, but only if path isn't a URL (anymore) -> special case,\n        # handles below\n        if ds is None and path is not None and not is_url(path):\n            # try to find a dataset at or above the installation target\n            dspath = GitRepo.get_toppath(abspath(path))\n            if dspath is None:\n                # no top-level dataset found, use path as such\n                dspath = path\n            ds = Dataset(dspath)\n\n        if ds is None and source is None and path is not None:\n            # no dataset, no source\n            # this could be a shortcut install call, where the first\n            # arg identifies the source\n            if is_url(path) or os.path.exists(path):\n                # we have an actual URL -> this should be the source\n                # OR\n                # it is not a URL, but it exists locally\n                lgr.debug(\n                    \"Single argument given to install and no dataset found. \"\n                    \"Assuming the argument identifies a source location.\")\n                source = path\n                path = None\n\n        lgr.debug(\"Resolved installation target: {0}\".format(path))\n\n        if ds is None and path is None and source is not None:\n            # we got nothing but a source. do something similar to git clone\n            # and derive the path from the source and continue\n            lgr.debug(\n                \"Neither dataset not target installation path provided. \"\n                \"Assuming installation of a remote dataset. \"\n                \"Deriving destination path from given source {0}\".format(\n                    source))\n            ds = Dataset(_installationpath_from_url(source))\n\n        if not path and ds is None:\n            # no dataset, no target location, nothing to do\n            raise InsufficientArgumentsError(\n                \"insufficient information for installation (needs at \"\n                \"least a dataset or an installation path\")\n\n        assert(ds is not None)\n\n        lgr.info(\"Installing {0}\".format(path if path else ds))\n\n        vcs = ds.repo\n        if vcs is None:\n            # TODO check that a \"ds.path\" actually points to a TOPDIR\n            # should be the case already, but maybe nevertheless check\n            existed = ds.path and exists(ds.path)\n\n            # We possibly need to consider /.git URL\n            candidate_sources = [source]\n            if source and not source.rstrip('/').endswith('/.git'):\n                candidate_sources.append('{0}/.git'.format(source.rstrip('/')))\n\n            for source_ in candidate_sources:\n                try:\n                    lgr.debug(\"Retrieving a dataset from URL: {0}\".format(source_))\n                    with swallow_logs():\n                        vcs = Install._get_new_vcs(ds, source_)\n                    break  # do not bother with other sources if succeeded\n                except GitCommandError:\n                    lgr.debug(\"Failed to retrieve from URL: {0}\".format(source_))\n                    if not existed and ds.path and exists(ds.path):\n                        lgr.debug(\"Wiping out unsuccessful clone attempt at {}\".format(ds.path))\n                        rmtree(ds.path)\n                    if source_ == candidate_sources[-1]:\n                        # Re-raise if failed even with the last candidate\n                        lgr.debug(\"Unable to establish repository instance at {0} from {1}\"\n                                  \"\".format(ds.path, candidate_sources))\n                        raise\n\n        assert(ds.repo)  # is automagically re-evaluated in the .repo property\n\n        runner = Runner()\n\n        if path is None or path == ds.path:\n            # if the goal was to install this dataset, we are done,\n            # except for 'recursive'.\n\n            # TODO: For now 'recursive' means just submodules.\n            # See --with-data vs. -- recursive and figure it out\n            if recursive:\n                for sm in ds.repo.get_submodules():\n                    _install_subds_from_flexible_source(\n                        ds, sm.path, sm.url, recursive=recursive)\n            return ds\n\n        # at this point this dataset is \"installed\", now we can test whether to\n        # install something into the dataset\n\n        # needed by the logic below\n        assert(isabs(path))\n\n        # express the destination path relative to the root of this dataset\n        relativepath = relpath(path, start=ds.path)\n        if path.startswith(pardir):\n            raise ValueError(\"installation path outside dataset\")\n\n        lgr.debug(\n            \"Resolved installation target relative to dataset {0}: {1}\".format(\n                ds, relativepath))\n\n        # this dataset must already know everything necessary\n        ###################################################\n        # FLOW GUIDE\n        #\n        # at this point we know nothing about the\n        # installation targether\n        ###################################################\n        try:\n            # it is simplest to let annex tell us what we are dealing with\n            lgr.debug(\"Trying to fetch file %s using annex\", relativepath)\n            if not isinstance(vcs, AnnexRepo):\n                assert(isinstance(vcs, GitRepo))\n                # FLOW GUIDE\n                # this is not an annex repo, but we raise exceptions\n                # to be able to treat them alike in the special case handling\n                # below\n                if not exists(path):\n                    raise IOError(\"path doesn't exist yet, might need special handling\")\n                elif relativepath in vcs.get_indexed_files():\n                    # relativepath is in git\n                    raise FileInGitError(\"We need to handle it as known to git\")\n                else:\n                    raise FileNotInAnnexError(\"We don't have yet annex repo here\")\n            if vcs.get_file_key(relativepath):\n                # FLOW GUIDE EXIT POINT\n                # this is an annex'ed file -> get it\n                # TODO implement `copy --from` using `source`\n                # TODO fail if `source` is something strange\n                vcs.get(relativepath)\n                # return the absolute path to the installed file\n                return path\n\n        except FileInGitError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - a  file already checked into Git\n            # - known submodule\n            ###################################################\n            lgr.log(5, \"FileInGitError logic\")\n            if source is not None:\n                raise FileInGitError(\"File %s is already in git. Specifying source (%s) makes no sense\"\n                                     % (path, source))\n            # file is checked into git directly -> nothing to do\n            # OR this is a submodule of this dataset\n            submodule = [sm for sm in ds.repo.get_submodules()\n                         if sm.path == relativepath]\n            if not len(submodule):\n                # FLOW GUIDE EXIT POINT\n                # this is a file in Git and no submodule, just return its path\n                lgr.debug(\"Don't act, data already present in Git\")\n                return path\n            elif len(submodule) > 1:\n                raise RuntimeError(\n                    \"more than one submodule registered at the same path?\")\n            submodule = submodule[0]\n\n            # FLOW GUIDE EXIT POINT\n            # we are dealing with a known submodule (i.e. `source`\n            # doesn't matter) -> check it out\n            lgr.debug(\"Install subdataset at: {0}\".format(submodule.path))\n            subds = _install_subds_from_flexible_source(\n                ds, submodule.path, submodule.url, recursive=recursive)\n            return subds\n\n        except FileNotInAnnexError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # `path` is either\n            # - content of a subdataset\n            # - an untracked file in this dataset\n            # - an entire untracked/unknown existing subdataset\n            ###################################################\n            lgr.log(5, \"FileNotInAnnexError logic\")\n            subds = get_containing_subdataset(ds, relativepath)\n            if ds.path != subds.path:\n                # FLOW GUIDE EXIT POINT\n                # target path belongs to a known subdataset, hand\n                # installation over to it\n                return subds.install(\n                    path=relpath(path, start=subds.path),\n                    source=source,\n                    recursive=recursive,\n                    add_data_to_git=add_data_to_git)\n\n            # FLOW GUIDE\n            # this must be an untracked/existing something, so either\n            # - a file\n            # - a directory\n            # - an entire repository\n            if exists(opj(path, '.git')):\n                # FLOW GUIDE EXIT POINT\n                # this is an existing repo and must be in-place turned into\n                # a submodule of this dataset\n                return _install_subds_inplace(\n                    ds, path, relativepath)\n\n            # FLOW GUIDE EXIT POINT\n            # - untracked file or directory in this dataset\n            if isdir(path) and not recursive:\n                # this is a directory and we want --recursive for it\n                raise ValueError(\n                    \"installation of a directory requires the `recursive` flag\")\n\n            # few sanity checks\n            if source and abspath(source) != path:\n                raise ValueError(\n                    \"installation target already exists, but `source` points to \"\n                    \"another location (target: '{0}', source: '{0}'\".format(\n                        source, path))\n\n            if not add_data_to_git and not (isinstance(vcs, AnnexRepo)):\n                raise RuntimeError(\n                    \"Trying to install file(s) into a dataset \"\n                    \"with a plain Git repository. First initialize annex, or \"\n                    \"provide override flag.\")\n\n            # switch `add` procedure between Git and Git-annex according to flag\n            if add_data_to_git:\n                vcs.add(relativepath, git=True)\n                added_files = resolve_path(relativepath, ds)\n            else:\n                # do a blunt `annex add`\n                added_files = vcs.add(relativepath)\n                # return just the paths of the installed components\n                if isinstance(added_files, list):\n                    added_files = [resolve_path(i['file'], ds) for i in added_files]\n                else:\n                    added_files = resolve_path(added_files['file'], ds)\n            if added_files:\n                return added_files\n            else:\n                return None\n\n        except IOError:\n            ###################################################\n            # FLOW GUIDE\n            #\n            # more complicated special cases -- `path` is either\n            # - a file/subdataset in a not yet initialized but known\n            #   submodule\n            # - an entire untracked/unknown existing subdataset\n            # - non-existing content that should be installed from `source`\n            ###################################################\n            lgr.log(5, \"IOError logic\")\n            # we can end up here in two cases ATM\n            if (exists(path) or islink(path)) or source is None:\n                # FLOW GUIDE\n                # - target exists but this dataset's VCS rejects it,\n                #   so it should be part of a subdataset\n                # or\n                # - target doesn't exist, but no source is given, so\n                #   it could be a handle that is actually contained in\n                #   a not yet installed subdataset\n                subds = get_containing_subdataset(ds, relativepath)\n                if ds.path != subds.path:\n                    # FLOW GUIDE\n                    # target path belongs to a subdataset, hand installation\n                    # over to it\n                    if not subds.is_installed():\n                        # FLOW GUIDE\n                        # we are dealing with a target in a not yet\n                        # available but known subdataset -> install it first\n                        ds.install(subds.path, recursive=recursive)\n                    return subds.install(\n                        path=relpath(path, start=subds.path),\n                        source=source,\n                        recursive=recursive,\n                        add_data_to_git=add_data_to_git)\n\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            if not source:\n                # FLOW GUIDE EXIT POINT\n                raise InsufficientArgumentsError(\n                    \"insufficient information for installation: the \"\n                    \"installation target {0} doesn't exists, isn't a \"\n                    \"known part of dataset {1}, and no `source` \"\n                    \"information was provided.\".format(path, ds))\n\n            source_path = expandpath(source)\n            if exists(source_path):\n                # FLOW GUIDE EXIT POINT\n                # this could be\n                # - local file\n                # - local directory\n                # - repository outside the dataset\n                # we only want to support the last case of locally cloning\n                # a repo -- fail otherwise\n                if exists(opj(source_path, '.git')):\n                    return _install_subds_from_flexible_source(\n                        ds, relativepath, source_path, recursive)\n\n                raise ValueError(\n                    \"installing individual local files or directories is not \"\n                    \"supported, copy/move them into the dataset first\")\n\n            # FLOW GUIDE\n            # `source` is non-local, it could be:\n            #   - repository\n            #   - file\n            # we have no further evidence, hence we need to try\n            try:\n                # FLOW GUIDE EXIT POINT\n                # assume it is a dataset\n                return _install_subds_from_flexible_source(\n                    ds, relativepath, source, recursive)\n            except CommandError:\n                # FLOW GUIDE EXIT POINT\n                # apaarently not a repo, assume it is a file url\n                vcs.add_url_to_file(relativepath, source)\n                return path\n\n    @staticmethod\n    def _get_new_vcs(ds, source):\n        if source is None:\n            raise RuntimeError(\n                \"No `source` was provided. To create a new dataset \"\n                \"use the `create` command.\")\n        else:\n            # when obtained from remote, try with plain Git\n            lgr.info(\"Creating a new git repo at %s\", ds.path)\n            vcs = GitRepo(ds.path, url=source, create=True)\n            if knows_annex(ds.path):\n                # init annex when traces of a remote annex can be detected\n                lgr.info(\"Initializing annex repo at %s\", ds.path)\n                vcs = AnnexRepo(ds.path, init=True)\n            else:\n                lgr.debug(\"New repository clone has no traces of an annex\")\n        return vcs\n\n    @staticmethod\n    def result_renderer_cmdline(res):\n        from datalad.ui import ui\n        if res is None:\n            res = []\n        if not isinstance(res, list):\n            res = [res]\n        if not len(res):\n            ui.message(\"Nothing was installed\")\n            return\n        items = '\\n'.join(map(str, res))\n        msg = \"{n} installed {obj} available at\\n{items}\".format(\n            obj='items are' if len(res) > 1 else 'item is',\n            n=len(res),\n            items=items)\n        ui.message(msg)",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 222,
                            "end_lineno": 667,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport sys\nimport re\n\nfrom ..ui import ui\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups():\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    return grps\n\n\ndef dedent_docstring(text):\n    import textwrap\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Command line use only \\>\\>.*\\<\\< Command line use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[CMD:\\s.*CMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Python use only \\>\\>.*\\<\\< Python use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s.*PY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    return docs\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from inspect import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if not defaults is None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if not param.constraints is None:\n                    param.constraints(defaults[defaults_idx])\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value contraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argnames = getargspec(cls.__call__)[0]\n        kwargs = {k: getattr(args, k) for k in argnames if k != 'self'}\n        try:\n            return cls.__call__(**kwargs)\n        except KeyboardInterrupt:\n            ui.error(\"\\nInterrupted by user while doing magic\")\n            sys.exit(1)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport sys\nimport re\n\nfrom ..ui import ui\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups():\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    return grps\n\n\ndef dedent_docstring(text):\n    import textwrap\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Command line use only \\>\\>.*\\<\\< Command line use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[CMD:\\s.*CMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Python use only \\>\\>.*\\<\\< Python use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s.*PY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    return docs\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from inspect import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if not defaults is None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if not param.constraints is None:\n                    param.constraints(defaults[defaults_idx])\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value contraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argnames = getargspec(cls.__call__)[0]\n        kwargs = {k: getattr(args, k) for k in argnames if k != 'self'}\n        try:\n            return cls.__call__(**kwargs)\n        except KeyboardInterrupt:\n            ui.error(\"\\nInterrupted by user while doing magic\")\n            sys.exit(1)\n",
            "header": {
                "file": {
                    "path": "datalad/interface/base.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "810dbead6",
                "new_commit_hash": "35aba6bed",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/base.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/base.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 110,
                            "length": 6
                        },
                        "new_range": {
                            "start": 110,
                            "length": 12
                        },
                        "section": "def alter_interface_docs_for_cmdline(docs):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "        '',",
                                "type": "context"
                            },
                            {
                                "content": "        docs,",
                                "type": "context"
                            },
                            {
                                "content": "        flags=re.MULTILINE | re.DOTALL)",
                                "type": "context"
                            },
                            {
                                "content": "    # remove :role:`...` RST markup for cmdline docs",
                                "type": "added"
                            },
                            {
                                "content": "    docs = re.sub(",
                                "type": "added"
                            },
                            {
                                "content": "        r':\\S+:`[^`]*`[\\\\]*',",
                                "type": "added"
                            },
                            {
                                "content": "        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),",
                                "type": "added"
                            },
                            {
                                "content": "        docs,",
                                "type": "added"
                            },
                            {
                                "content": "        flags=re.MULTILINE | re.DOTALL)",
                                "type": "added"
                            },
                            {
                                "content": "    # remove None constraint. In general, `None` on the cmdline means don't",
                                "type": "context"
                            },
                            {
                                "content": "    # give option at all, but specifying `None` explicitly is practically",
                                "type": "context"
                            },
                            {
                                "content": "    # impossible",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/base.py",
                            "type": "function",
                            "name": "alter_interface_docs_for_cmdline",
                            "content": "def alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Python use only \\>\\>.*\\<\\< Python use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s.*PY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove :role:`...` RST markup for cmdline docs\n    docs = re.sub(\n        r':\\S+:`[^`]*`[\\\\]*',\n        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\\\'),\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    return docs",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 94,
                            "end_lineno": 133,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for interface doc wranglers.\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom datalad.interface.base import dedent_docstring\nfrom datalad.interface.base import alter_interface_docs_for_api\nfrom datalad.interface.base import alter_interface_docs_for_cmdline\nfrom datalad.tests.utils import assert_true, assert_false, assert_in, \\\n    assert_not_in\n\n\ndemo_doc = \"\"\"\\\n    Bla bla summary\n\n    Generic intro blurb. Ping pong ping pong ping pong ping pong.  Ping pong ping\n    pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong\n    ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping\n    pong ping pong ping pong ping pong.\n\n    || Command line use only >>\n    Something for the cmdline only CMDONLY\n    Multiline!\n    << Command line use only ||\n\n    || Python use only >>\n\n    Some Python-only bits PYONLY\n    Multiline!\n\n    << Python use only ||\n\n    And an example for in-line markup: [PY: just for Python PY] and\n    the other one [CMD: just for the command line CMD]. End of demo.\n\n    Generic appendix. Ding dong ding dong ding dong.  Ding dong ding dong ding\n    dong.  Ding dong ding dong ding dong.  Ding dong ding dong ding dong.  Ding\n    dong ding dong ding dong.\n\n\"\"\"\n\ndemo_paramdoc = \"\"\"\\\n\n    Parameters\n    ----------\n    dataset : Dataset or None, optional\n      specify the dataset to perform the install operation on. If no\n      dataset is given, an attempt is made to identify the dataset based\n      on the current working directory and/or the `path` given.\n      Constraints: Value must be a Dataset or a valid identifier of a\n      Dataset (e.g. a path), or value must be `None`. [Default: None]\n\"\"\"\n\ndemo_argdoc = \"\"\"\\\n    specify the dataset to perform the install operation\n    on. If no dataset is given, an attempt is made to\n    identify the dataset based on the current working\n    directory and/or the `path` given. Constraints: Value\n    must be a Dataset or a valid identifier of a Dataset\n    (e.g. a path), or value must be `None`. [Default:\n    None]\n\"\"\"\n\n\ndef test_dedent():\n    assert_false(dedent_docstring(\"one liner\").endswith(\"\\n\"))\n\n\ndef test_alter_interface_docs_for_api():\n    alt = alter_interface_docs_for_api(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('CMD', alt)\n    assert_not_in('Command line', alt)\n\n\ndef test_alter_interface_docs_for_cmdline():\n    alt = alter_interface_docs_for_cmdline(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('PY', alt)\n    assert_not_in('Python', alt)\n    # args\n    altarg = alter_interface_docs_for_cmdline(demo_argdoc)\n    # TO BE CONTINUED\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for interface doc wranglers.\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom datalad.interface.base import dedent_docstring\nfrom datalad.interface.base import alter_interface_docs_for_api\nfrom datalad.interface.base import alter_interface_docs_for_cmdline\nfrom datalad.tests.utils import assert_true, assert_false, assert_in, \\\n    assert_not_in, eq_\n\n\ndemo_doc = \"\"\"\\\n    Bla bla summary\n\n    Generic intro blurb. Ping pong ping pong ping pong ping pong.  Ping pong ping\n    pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong\n    ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping\n    pong ping pong ping pong ping pong.\n\n    || Command line use only >>\n    Something for the cmdline only CMDONLY\n    Multiline!\n    << Command line use only ||\n\n    || Python use only >>\n\n    Some Python-only bits PYONLY\n    Multiline!\n\n    << Python use only ||\n\n    And an example for in-line markup: [PY: just for Python PY] and\n    the other one [CMD: just for the command line CMD]. End of demo.\n\n    Generic appendix. Ding dong ding dong ding dong.  Ding dong ding dong ding\n    dong.  Ding dong ding dong ding dong.  Ding dong ding dong ding dong.  Ding\n    dong ding dong ding dong.\n\n\"\"\"\n\ndemo_paramdoc = \"\"\"\\\n\n    Parameters\n    ----------\n    dataset : Dataset or None, optional\n      specify the dataset to perform the install operation on. If no\n      dataset is given, an attempt is made to identify the dataset based\n      on the current working directory and/or the `path` given.\n      Constraints: Value must be a Dataset or a valid identifier of a\n      Dataset (e.g. a path), or value must be `None`. [Default: None]\n\"\"\"\n\ndemo_argdoc = \"\"\"\\\n    specify the dataset to perform the install operation\n    on. If no dataset is given, an attempt is made to\n    identify the dataset based on the current working\n    directory and/or the `path` given. Constraints: Value\n    must be a Dataset or a valid identifier of a Dataset\n    (e.g. a path), or value must be `None`. [Default:\n    None]\n\"\"\"\n\n\ndef test_dedent():\n    assert_false(dedent_docstring(\"one liner\").endswith(\"\\n\"))\n\n\ndef test_alter_interface_docs_for_api():\n    alt = alter_interface_docs_for_api(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('CMD', alt)\n    assert_not_in('Command line', alt)\n\n\ndef test_alter_interface_docs_for_cmdline():\n    alt = alter_interface_docs_for_cmdline(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('PY', alt)\n    assert_not_in('Python', alt)\n    # args\n    altarg = alter_interface_docs_for_cmdline(demo_argdoc)\n    # RST role markup\n    eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'),\n        'me and my buddies')\n    # spread across lines\n    eq_(alter_interface_docs_for_cmdline(':term:`Barbara\\nStreisand`'),\n        'Barbara\\nStreisand')\n    # multiple on one line\n    eq_(alter_interface_docs_for_cmdline(\n        ':term:`one` bla bla :term:`two` bla'),\n        'one bla bla two bla')\n",
            "header": {
                "file": {
                    "path": "datalad/interface/tests/test_docs.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "72a9f1f34",
                "new_commit_hash": "0c8b10b16",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/interface/tests/test_docs.py"
            },
            "plus_file": {
                "path": "b/datalad/interface/tests/test_docs.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 16,
                            "length": 7
                        },
                        "new_range": {
                            "start": 16,
                            "length": 7
                        },
                        "section": "from datalad.interface.base import dedent_docstring"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "from datalad.interface.base import alter_interface_docs_for_api",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.interface.base import alter_interface_docs_for_cmdline",
                                "type": "context"
                            },
                            {
                                "content": "from datalad.tests.utils import assert_true, assert_false, assert_in, \\",
                                "type": "context"
                            },
                            {
                                "content": "    assert_not_in",
                                "type": "deleted"
                            },
                            {
                                "content": "    assert_not_in, eq_",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "demo_doc = \"\"\"\\",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_docs.py",
                            "type": "statement",
                            "name": "",
                            "content": "from datalad.tests.utils import assert_true, assert_false, assert_in, \\\n    assert_not_in, eq_",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 18,
                            "end_lineno": 19,
                            "parent": null
                        }
                    ],
                    "deleted_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_docs.py",
                            "type": "statement",
                            "name": "",
                            "content": "from datalad.tests.utils import assert_true, assert_false, assert_in, \\\n    assert_not_in",
                            "ast_type_str": "ImportFrom",
                            "start_lineno": 18,
                            "end_lineno": 19,
                            "parent": null
                        }
                    ]
                },
                {
                    "descriptor": {
                        "old_range": {
                            "start": 95,
                            "length": 4
                        },
                        "new_range": {
                            "start": 95,
                            "length": 13
                        },
                        "section": "def test_alter_interface_docs_for_cmdline():"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "    assert_not_in('Python', alt)",
                                "type": "context"
                            },
                            {
                                "content": "    # args",
                                "type": "context"
                            },
                            {
                                "content": "    altarg = alter_interface_docs_for_cmdline(demo_argdoc)",
                                "type": "context"
                            },
                            {
                                "content": "    # TO BE CONTINUED",
                                "type": "deleted"
                            },
                            {
                                "content": "    # RST role markup",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'),",
                                "type": "added"
                            },
                            {
                                "content": "        'me and my buddies')",
                                "type": "added"
                            },
                            {
                                "content": "    # spread across lines",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(alter_interface_docs_for_cmdline(':term:`Barbara\\nStreisand`'),",
                                "type": "added"
                            },
                            {
                                "content": "        'Barbara\\nStreisand')",
                                "type": "added"
                            },
                            {
                                "content": "    # multiple on one line",
                                "type": "added"
                            },
                            {
                                "content": "    eq_(alter_interface_docs_for_cmdline(",
                                "type": "added"
                            },
                            {
                                "content": "        ':term:`one` bla bla :term:`two` bla'),",
                                "type": "added"
                            },
                            {
                                "content": "        'one bla bla two bla')",
                                "type": "added"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/interface/tests/test_docs.py",
                            "type": "function",
                            "name": "test_alter_interface_docs_for_cmdline",
                            "content": "def test_alter_interface_docs_for_cmdline():\n    alt = alter_interface_docs_for_cmdline(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('PY', alt)\n    assert_not_in('Python', alt)\n    # args\n    altarg = alter_interface_docs_for_cmdline(demo_argdoc)\n    # RST role markup\n    eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'),\n        'me and my buddies')\n    # spread across lines\n    eq_(alter_interface_docs_for_cmdline(':term:`Barbara\\nStreisand`'),\n        'Barbara\\nStreisand')\n    # multiple on one line\n    eq_(alter_interface_docs_for_cmdline(\n        ':term:`one` bla bla :term:`two` bla'),\n        'one bla bla two bla')",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 88,
                            "end_lineno": 107,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "",
            "new_file_content": ".. -*- mode: rst; fill-column: 78; indent-tabs-mode: nil -*-\n.. vi: set ft=rst sts=4 ts=4 sw=4 et tw=79:\n  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###\n  #\n  #   See COPYING file distributed along with the datalad package for the\n  #   copyright and license terms.\n  #\n  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###\n\n.. _chap_glossary:\n\n********\nGlossary\n********\n\nDatalad purposefully uses a terminology that is different from the one used by\nits technological foundations Git_ and git-annex_. This glossary provides\ndefinitions for terms used in the datalad documentation and API, and relates\nthem to the corresponding Git_/git-annex_ concepts.\n\n.. glossary::\n  :sorted:\n\n  dataset\n    A regular Git_ repository with an (optional) :term:`annex`.\n\n  subdataset\n    A :term:`dataset` that is part of another dataset, by means of being\n    tracked as a Git_ submodule. As such, a subdataset is also a complete\n    dataset and not different from a standalone dataset.\n\n  superdataset\n    A :term:`dataset` that contains at least one :term:`subdataset`.\n\n  sibling\n    A :term:`dataset` (location) that is related to a particular dataset,\n    by sharing content and history. In Git_ terminology, this is a *clone*\n    of a dataset that is configured as a *remote*.\n\n  annex\n    Extension to a Git_ repository, provided and managed by git-annex_ as\n    means to track and distribute large (and small) files without having to\n    inject them directly into a Git_ repository (which would slow Git\n    operations significantly and impair handling of such repositories in\n    general).\n\n.. _Git: https://git-scm.com\n.. _Git-annex: http://git-annex.branchable.com\n",
            "header": {
                "file": {
                    "path": "docs/source/glossary.rst"
                },
                "misc_line": "new file mode 100644"
            },
            "index_line": {
                "old_commit_hash": "000000000",
                "new_commit_hash": "4cd4b6274",
                "mode": ""
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "/dev/null"
            },
            "plus_file": {
                "path": "b/docs/source/glossary.rst"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 0,
                            "length": 0
                        },
                        "new_range": {
                            "start": 1,
                            "length": 48
                        },
                        "section": ""
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": ".. -*- mode: rst; fill-column: 78; indent-tabs-mode: nil -*-",
                                "type": "added"
                            },
                            {
                                "content": ".. vi: set ft=rst sts=4 ts=4 sw=4 et tw=79:",
                                "type": "added"
                            },
                            {
                                "content": "  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###",
                                "type": "added"
                            },
                            {
                                "content": "  #",
                                "type": "added"
                            },
                            {
                                "content": "  #   See COPYING file distributed along with the datalad package for the",
                                "type": "added"
                            },
                            {
                                "content": "  #   copyright and license terms.",
                                "type": "added"
                            },
                            {
                                "content": "  #",
                                "type": "added"
                            },
                            {
                                "content": "  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": ".. _chap_glossary:",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "********",
                                "type": "added"
                            },
                            {
                                "content": "Glossary",
                                "type": "added"
                            },
                            {
                                "content": "********",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "Datalad purposefully uses a terminology that is different from the one used by",
                                "type": "added"
                            },
                            {
                                "content": "its technological foundations Git_ and git-annex_. This glossary provides",
                                "type": "added"
                            },
                            {
                                "content": "definitions for terms used in the datalad documentation and API, and relates",
                                "type": "added"
                            },
                            {
                                "content": "them to the corresponding Git_/git-annex_ concepts.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": ".. glossary::",
                                "type": "added"
                            },
                            {
                                "content": "  :sorted:",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "  dataset",
                                "type": "added"
                            },
                            {
                                "content": "    A regular Git_ repository with an (optional) :term:`annex`.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "  subdataset",
                                "type": "added"
                            },
                            {
                                "content": "    A :term:`dataset` that is part of another dataset, by means of being",
                                "type": "added"
                            },
                            {
                                "content": "    tracked as a Git_ submodule. As such, a subdataset is also a complete",
                                "type": "added"
                            },
                            {
                                "content": "    dataset and not different from a standalone dataset.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "  superdataset",
                                "type": "added"
                            },
                            {
                                "content": "    A :term:`dataset` that contains at least one :term:`subdataset`.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "  sibling",
                                "type": "added"
                            },
                            {
                                "content": "    A :term:`dataset` (location) that is related to a particular dataset,",
                                "type": "added"
                            },
                            {
                                "content": "    by sharing content and history. In Git_ terminology, this is a *clone*",
                                "type": "added"
                            },
                            {
                                "content": "    of a dataset that is configured as a *remote*.",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": "  annex",
                                "type": "added"
                            },
                            {
                                "content": "    Extension to a Git_ repository, provided and managed by git-annex_ as",
                                "type": "added"
                            },
                            {
                                "content": "    means to track and distribute large (and small) files without having to",
                                "type": "added"
                            },
                            {
                                "content": "    inject them directly into a Git_ repository (which would slow Git",
                                "type": "added"
                            },
                            {
                                "content": "    operations significantly and impair handling of such repositories in",
                                "type": "added"
                            },
                            {
                                "content": "    general).",
                                "type": "added"
                            },
                            {
                                "content": "",
                                "type": "added"
                            },
                            {
                                "content": ".. _Git: https://git-scm.com",
                                "type": "added"
                            },
                            {
                                "content": ".. _Git-annex: http://git-annex.branchable.com",
                                "type": "added"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": ".. datalad documentation master file, created by\n   sphinx-quickstart on Tue Oct 13 08:41:19 2015.\n   You can adapt this file completely to your liking, but it should at least\n   contain the root `toctree` directive.\n\nWelcome to datalad's documentation!\n===================================\n\nContents:\n\n.. toctree::\n   :maxdepth: 2\n\n   usecases/index\n   crawler/index\n   cmdline\n   modref\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n\n",
            "new_file_content": ".. datalad documentation master file, created by\n   sphinx-quickstart on Tue Oct 13 08:41:19 2015.\n   You can adapt this file completely to your liking, but it should at least\n   contain the root `toctree` directive.\n\nWelcome to datalad's documentation!\n===================================\n\nContents:\n\n.. toctree::\n   :maxdepth: 2\n\n   usecases/index\n   crawler/index\n   cmdline\n   glossary\n   modref\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n\n",
            "header": {
                "file": {
                    "path": "docs/source/index.rst"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "684551df6",
                "new_commit_hash": "0329f7af7",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/docs/source/index.rst"
            },
            "plus_file": {
                "path": "b/docs/source/index.rst"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 14,
                            "length": 6
                        },
                        "new_range": {
                            "start": 14,
                            "length": 7
                        },
                        "section": "Contents:"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "   usecases/index",
                                "type": "context"
                            },
                            {
                                "content": "   crawler/index",
                                "type": "context"
                            },
                            {
                                "content": "   cmdline",
                                "type": "context"
                            },
                            {
                                "content": "   glossary",
                                "type": "added"
                            },
                            {
                                "content": "   modref",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "Indices and tables",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "9827892da941857c0dab1e68ab3d8f8c9bf3b8a8^",
    "new_commit_hash": "9827892da941857c0dab1e68ab3d8f8c9bf3b8a8",
    "commit_message": "Merge pull request #560 from datalad/docs\n\nBetter docs",
    "commit_date": "2016-06-17T11:05:16+02:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "9827892da941857c0dab1e68ab3d8f8c9bf3b8a8",
    "test_file_codes": [
        "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Tests for interface doc wranglers.\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nfrom datalad.interface.base import dedent_docstring\nfrom datalad.interface.base import alter_interface_docs_for_api\nfrom datalad.interface.base import alter_interface_docs_for_cmdline\nfrom datalad.tests.utils import assert_true, assert_false, assert_in, \\\n    assert_not_in, eq_\n\n\ndemo_doc = \"\"\"\\\n    Bla bla summary\n\n    Generic intro blurb. Ping pong ping pong ping pong ping pong.  Ping pong ping\n    pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping pong\n    ping pong ping pong ping pong. Ping pong ping pong ping pong ping pong. Ping\n    pong ping pong ping pong ping pong.\n\n    || Command line use only >>\n    Something for the cmdline only CMDONLY\n    Multiline!\n    << Command line use only ||\n\n    || Python use only >>\n\n    Some Python-only bits PYONLY\n    Multiline!\n\n    << Python use only ||\n\n    And an example for in-line markup: [PY: just for Python PY] and\n    the other one [CMD: just for the command line CMD]. End of demo.\n\n    Generic appendix. Ding dong ding dong ding dong.  Ding dong ding dong ding\n    dong.  Ding dong ding dong ding dong.  Ding dong ding dong ding dong.  Ding\n    dong ding dong ding dong.\n\n\"\"\"\n\ndemo_paramdoc = \"\"\"\\\n\n    Parameters\n    ----------\n    dataset : Dataset or None, optional\n      specify the dataset to perform the install operation on. If no\n      dataset is given, an attempt is made to identify the dataset based\n      on the current working directory and/or the `path` given.\n      Constraints: Value must be a Dataset or a valid identifier of a\n      Dataset (e.g. a path), or value must be `None`. [Default: None]\n\"\"\"\n\ndemo_argdoc = \"\"\"\\\n    specify the dataset to perform the install operation\n    on. If no dataset is given, an attempt is made to\n    identify the dataset based on the current working\n    directory and/or the `path` given. Constraints: Value\n    must be a Dataset or a valid identifier of a Dataset\n    (e.g. a path), or value must be `None`. [Default:\n    None]\n\"\"\"\n\n\ndef test_dedent():\n    assert_false(dedent_docstring(\"one liner\").endswith(\"\\n\"))\n\n\ndef test_alter_interface_docs_for_api():\n    alt = alter_interface_docs_for_api(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('CMD', alt)\n    assert_not_in('Command line', alt)\n\n\ndef test_alter_interface_docs_for_cmdline():\n    alt = alter_interface_docs_for_cmdline(demo_doc)\n    alt_l = alt.split('\\n')\n    # dedented\n    assert_false(alt_l[0].startswith(' '))\n    assert_false(alt_l[-1].startswith(' '))\n    assert_not_in('PY', alt)\n    assert_not_in('Python', alt)\n    # args\n    altarg = alter_interface_docs_for_cmdline(demo_argdoc)\n    # RST role markup\n    eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'),\n        'me and my buddies')\n    # spread across lines\n    eq_(alter_interface_docs_for_cmdline(':term:`Barbara\\nStreisand`'),\n        'Barbara\\nStreisand')\n    # multiple on one line\n    eq_(alter_interface_docs_for_cmdline(\n        ':term:`one` bla bla :term:`two` bla'),\n        'one bla bla two bla')\n",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.2.1.dev96\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 129ms\nInstalled 25 packages in 6.19s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpUDRdtT/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpUDRdtT/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpUDRdtT/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpUDRdtT/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpUDRdtT/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 39, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpo0MFG7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpo0MFG7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpo0MFG7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpo0MFG7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpo0MFG7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 39, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8/datalad/__init__.py\", line 20, in <module>\n    from .config import ConfigManager\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8/datalad/config.py\", line 17, in <module>\n    from appdirs import AppDirs\nModuleNotFoundError: No module named 'appdirs'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 396ms\nInstalled 26 packages in 5.57s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 67 packages in 2.59s\nPrepared 1 package in 1.08s\nInstalled 61 packages in 56ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + charset-normalizer==3.4.1\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.2.1.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8)\n + filelock==3.12.2\n + gitdb==4.0.12\n + gitpython==3.1.44\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack-python==0.5.6\n + multidict==6.0.5\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydispatcher==2.0.7\n + pyopenssl==24.3.0\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + smmap==5.0.2\n + soupsieve==2.4.1\n + testtools==2.7.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nAudited 1 package in 3ms\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8/.venv/lib/python3.7/site-packages/urllib3/__init__.py:38: NotOpenSSLWarning: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 3.1.4'. See: https://github.com/urllib3/urllib3/issues/3020\n  exceptions.NotOpenSSLWarning,\n",
    "new_commit_res_code": 0,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8\nplugins: cov-4.1.0\ncollected 3 items\n\nr2e_tests/test_1.py ...                                                  [100%]\n\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_dedent\nPASSED r2e_tests/test_1.py::test_alter_interface_docs_for_api\nPASSED r2e_tests/test_1.py::test_alter_interface_docs_for_cmdline\n============================== 3 passed in 1.05s ===============================\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_9827892da941857c0dab1e68ab3d8f8c9bf3b8a8\nplugins: cov-4.1.0\ncollected 3 items\n\nr2e_tests/test_1.py ..F                                                  [100%]\n\n=================================== FAILURES ===================================\n____________________ test_alter_interface_docs_for_cmdline _____________________\n\n    def test_alter_interface_docs_for_cmdline():\n        alt = alter_interface_docs_for_cmdline(demo_doc)\n        alt_l = alt.split('\\n')\n        # dedented\n        assert_false(alt_l[0].startswith(' '))\n        assert_false(alt_l[-1].startswith(' '))\n        assert_not_in('PY', alt)\n        assert_not_in('Python', alt)\n        # args\n        altarg = alter_interface_docs_for_cmdline(demo_argdoc)\n        # RST role markup\n        eq_(alter_interface_docs_for_cmdline(':murks:`me and my buddies`'),\n>           'me and my buddies')\n\nr2e_tests/test_1.py:100: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = ':murks:`me and my buddies`', b = 'me and my buddies', msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: ':murks:`me and my buddies`' != 'me and my buddies'\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n==================================== PASSES ====================================\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_dedent\nPASSED r2e_tests/test_1.py::test_alter_interface_docs_for_api\nFAILED r2e_tests/test_1.py::test_alter_interface_docs_for_cmdline - Assertion...\n========================= 1 failed, 2 passed in 0.41s ==========================\n",
    "old_commit_res_stderr": ""
}
datalad/distribution/install.py datalad/interface/base.py datalad/interface/tests/test_docs.py docs/source/glossary.rst docs/source/index.rst
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** RST Role Markup Not Stripped in Command Line Documentation

**Description:**
When generating command line documentation, RST role markup such as `:murks:` is not being removed properly. This results in documentation strings retaining unwanted RST syntax, which can lead to rendering issues or confusion.

**Example Buggy Code:**
```python
from datalad.interface.base import alter_interface_docs_for_cmdline

doc_string = ':murks:`me and my buddies`'
processed_doc = alter_interface_docs_for_cmdline(doc_string)
print(processed_doc)
```

**Expected Behavior:**
The `alter_interface_docs_for_cmdline` function should strip out the RST role markup, resulting in the following output:
```
me and my buddies
```

**Actual Behavior:**
The function retains the RST role markup, producing:
```
:murks:`me and my buddies`
```
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""High-level interface generation

"""

__docformat__ = 'restructuredtext'

import sys
import re

from ..ui import ui


def get_api_name(intfspec):
    """Given an interface specification return an API name for it"""
    if len(intfspec) > 3:
        name = intfspec[3]
    else:
        name = intfspec[0].split('.')[-1]
    return name


def get_cmdline_command_name(intfspec):
    """Given an interface specification return a cmdline command name"""
    if len(intfspec) > 2:
        name = intfspec[2]
    else:
        name = intfspec[0].split('.')[-1].replace('_', '-')
    return name


def get_interface_groups():
    from .. import interface as _interfaces

    grps = []
    # auto detect all available interfaces and generate a function-based
    # API from them
    for _item in _interfaces.__dict__:
        if not _item.startswith('_group_'):
            continue
        grp_name = _item[7:]
        grp = getattr(_interfaces, _item)
        grps.append((grp_name,) + grp)
    return grps


def dedent_docstring(text):
    import textwrap
    """Remove uniform indentation from a multiline docstring"""
    # Problem is that first line might often have no offset, so might
    # need to be ignored from dedent call
    if text is None:
        return None
    if not text.startswith(' '):
        lines = text.split('\n')
        if len(lines) == 1:
            # single line, no indentation, nothing to do
            return text
        text2 = '\n'.join(lines[1:])
        return lines[0] + "\n" + textwrap.dedent(text2)
    else:
        return textwrap.dedent(text)


def alter_interface_docs_for_api(docs):
    """Apply modifications to interface docstrings for Python API use."""
    # central place to alter the impression of docstrings,
    # like removing cmdline specific sections
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        '\|\| Command line use only \>\>.*\<\< Command line use only \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        '\[CMD:\s.*CMD\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def alter_interface_docs_for_cmdline(docs):
    """Apply modifications to interface docstrings for cmdline doc use."""
    # central place to alter the impression of docstrings,
    # like removing Python API specific sections, and argument markup
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        '\|\| Python use only \>\>.*\<\< Python use only \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        '\[PY:\s.*PY\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # remove None constraint. In general, `None` on the cmdline means don't
    # give option at all, but specifying `None` explicitly is practically
    # impossible
    docs = re.sub(
        ',\sor\svalue\smust\sbe\s`None`',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # capitalize variables and remove backticks to uniformize with
    # argparse output
    docs = re.sub(
        '`\S*`',
        lambda match: match.group(0).strip('`').upper(),
        docs)
    return docs


def update_docstring_with_parameters(func, params, prefix=None, suffix=None):
    """Generate a useful docstring from a parameter spec

    Amends any existing docstring of a callable with a textual
    description of its parameters. The Parameter spec needs to match
    the number and names of the callables arguments.
    """
    from inspect import getargspec
    # get the signature
    ndefaults = 0
    args, varargs, varkw, defaults = getargspec(func)
    if not defaults is None:
        ndefaults = len(defaults)
    # start documentation with what the callable brings with it
    doc = prefix if prefix else u''
    if len(args) > 1:
        if len(doc):
            doc += '\n'
        doc += "Parameters\n----------\n"
        for i, arg in enumerate(args):
            if arg == 'self':
                continue
            # we need a parameter spec for each argument
            if not arg in params:
                raise ValueError("function has argument '%s' not described as a parameter" % arg)
            param = params[arg]
            # validate the default -- to make sure that the parameter description is
            # somewhat OK
            defaults_idx = ndefaults - len(args) + i
            if defaults_idx >= 0:
                if not param.constraints is None:
                    param.constraints(defaults[defaults_idx])
            doc += param.get_autodoc(
                arg,
                default=defaults[defaults_idx] if defaults_idx >= 0 else None,
                has_default=defaults_idx >= 0)
            doc += '\n'
    doc += suffix if suffix else u""
    # assign the amended docs
    func.__doc__ = doc
    return func


class Interface(object):
    """Base class for interface implementations"""

    @classmethod
    def setup_parser(cls, parser):
        # XXX needs safety check for name collisions
        # XXX allow for parser kwargs customization
        parser_kwargs = {}
        from inspect import getargspec
        # get the signature
        ndefaults = 0
        args, varargs, varkw, defaults = getargspec(cls.__call__)
        if not defaults is None:
            ndefaults = len(defaults)
        for i, arg in enumerate(args):
            if arg == 'self':
                continue
            param = cls._params_[arg]
            defaults_idx = ndefaults - len(args) + i
            cmd_args = param.cmd_args
            if cmd_args is None:
                cmd_args = []
            if not len(cmd_args):
                if defaults_idx >= 0:
                    # dealing with a kwarg
                    template = '--%s'
                else:
                    # positional arg
                    template = '%s'
                # use parameter name as default argument name
                parser_args = (template % arg.replace('_', '-'),)
            else:
                parser_args = [c.replace('_', '-') for c in cmd_args]
            parser_kwargs = param.cmd_kwargs
            if defaults_idx >= 0:
                parser_kwargs['default'] = defaults[defaults_idx]
            help = alter_interface_docs_for_cmdline(param._doc)
            if help and help[-1] != '.':
                help += '.'
            if param.constraints is not None:
                parser_kwargs['type'] = param.constraints
                # include value contraint description and default
                # into the help string
                cdoc = alter_interface_docs_for_cmdline(
                    param.constraints.long_description())
                if cdoc[0] == '(' and cdoc[-1] == ')':
                    cdoc = cdoc[1:-1]
                help += '  Constraints: %s' % cdoc
            if defaults_idx >= 0:
                help += " [Default: %r]" % (defaults[defaults_idx],)
            # create the parameter, using the constraint instance for type
            # conversion
            parser.add_argument(*parser_args, help=help,
                                **parser_kwargs)

    @classmethod
    def call_from_parser(cls, args):
        # XXX needs safety check for name collisions
        from inspect import getargspec
        argnames = getargspec(cls.__call__)[0]
        kwargs = {k: getattr(args, k) for k in argnames if k != 'self'}
        try:
            return cls.__call__(**kwargs)
        except KeyboardInterrupt:
            ui.error("\nInterrupted by user while doing magic")
            sys.exit(1)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** RST Role Markup Not Stripped in Command Line Documentation

**Description:**
When generating command line documentation, RST role markup such as `:murks:` is not being removed properly. This results in documentation strings retaining unwanted RST syntax, which can lead to rendering issues or confusion.

**Example Buggy Code:**
```python
from datalad.interface.base import alter_interface_docs_for_cmdline

doc_string = ':murks:`me and my buddies`'
processed_doc = alter_interface_docs_for_cmdline(doc_string)
print(processed_doc)
```

**Expected Behavior:**
The `alter_interface_docs_for_cmdline` function should strip out the RST role markup, resulting in the following output:
```
me and my buddies
```

**Actual Behavior:**
The function retains the RST role markup, producing:
```
:murks:`me and my buddies`
```
[/ISSUE]
{
    "test_dedent": "PASSED",
    "test_alter_interface_docs_for_api": "PASSED",
    "test_alter_interface_docs_for_cmdline": "PASSED"
}
diff --git a/datalad/distribution/install.py b/datalad/distribution/install.py
index 04559ac8c..816881f39 100644
--- a/datalad/distribution/install.py
+++ b/datalad/distribution/install.py
@@ -223,9 +223,9 @@ class Install(Interface):
     """Install a dataset component or entire dataset(s).
 
     This command can make arbitrary content available in a dataset. This
-    includes the fulfillment of existing subdataset or file handles
-    in a dataset, as well as the addition of sudatasets and such handles for
-    content available locally or remotely.
+    includes the fulfillment of existing :term:`subdataset` or file handles in
+    a dataset, as well as the addition of subdatasets and such handles for
+    content available locally or subdataset remotely.
     """
 
     _params_ = dict(
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index 810dbead6..35aba6bed 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -110,6 +110,12 @@ def alter_interface_docs_for_cmdline(docs):
         '',
         docs,
         flags=re.MULTILINE | re.DOTALL)
+    # remove :role:`...` RST markup for cmdline docs
+    docs = re.sub(
+        r':\S+:`[^`]*`[\\]*',
+        lambda match: ':'.join(match.group(0).split(':')[2:]).strip('`\\'),
+        docs,
+        flags=re.MULTILINE | re.DOTALL)
     # remove None constraint. In general, `None` on the cmdline means don't
     # give option at all, but specifying `None` explicitly is practically
     # impossible

{"datalad/interface/base.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"High-level interface generation\n\n\"\"\"\n\n__docformat__ = 'restructuredtext'\n\nimport sys\nimport re\n\nfrom ..ui import ui\n\n\ndef get_api_name(intfspec):\n    \"\"\"Given an interface specification return an API name for it\"\"\"\n    if len(intfspec) > 3:\n        name = intfspec[3]\n    else:\n        name = intfspec[0].split('.')[-1]\n    return name\n\n\ndef get_cmdline_command_name(intfspec):\n    \"\"\"Given an interface specification return a cmdline command name\"\"\"\n    if len(intfspec) > 2:\n        name = intfspec[2]\n    else:\n        name = intfspec[0].split('.')[-1].replace('_', '-')\n    return name\n\n\ndef get_interface_groups():\n    from .. import interface as _interfaces\n\n    grps = []\n    # auto detect all available interfaces and generate a function-based\n    # API from them\n    for _item in _interfaces.__dict__:\n        if not _item.startswith('_group_'):\n            continue\n        grp_name = _item[7:]\n        grp = getattr(_interfaces, _item)\n        grps.append((grp_name,) + grp)\n    return grps\n\n\ndef dedent_docstring(text):\n    import textwrap\n    \"\"\"Remove uniform indentation from a multiline docstring\"\"\"\n    # Problem is that first line might often have no offset, so might\n    # need to be ignored from dedent call\n    if text is None:\n        return None\n    if not text.startswith(' '):\n        lines = text.split('\\n')\n        if len(lines) == 1:\n            # single line, no indentation, nothing to do\n            return text\n        text2 = '\\n'.join(lines[1:])\n        return lines[0] + \"\\n\" + textwrap.dedent(text2)\n    else:\n        return textwrap.dedent(text)\n\n\ndef alter_interface_docs_for_api(docs):\n    \"\"\"Apply modifications to interface docstrings for Python API use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing cmdline specific sections\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Command line use only \\>\\>.*\\<\\< Command line use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[CMD:\\s.*CMD\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    return docs\n\n\ndef alter_interface_docs_for_cmdline(docs):\n    \"\"\"Apply modifications to interface docstrings for cmdline doc use.\"\"\"\n    # central place to alter the impression of docstrings,\n    # like removing Python API specific sections, and argument markup\n    if not docs:\n        return docs\n    docs = dedent_docstring(docs)\n    # clean cmdline sections\n    docs = re.sub(\n        '\\|\\| Python use only \\>\\>.*\\<\\< Python use only \\|\\|',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # clean cmdline in-line bits\n    docs = re.sub(\n        '\\[PY:\\s.*PY\\]',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # remove None constraint. In general, `None` on the cmdline means don't\n    # give option at all, but specifying `None` explicitly is practically\n    # impossible\n    docs = re.sub(\n        ',\\sor\\svalue\\smust\\sbe\\s`None`',\n        '',\n        docs,\n        flags=re.MULTILINE | re.DOTALL)\n    # capitalize variables and remove backticks to uniformize with\n    # argparse output\n    docs = re.sub(\n        '`\\S*`',\n        lambda match: match.group(0).strip('`').upper(),\n        docs)\n    return docs\n\n\ndef update_docstring_with_parameters(func, params, prefix=None, suffix=None):\n    \"\"\"Generate a useful docstring from a parameter spec\n\n    Amends any existing docstring of a callable with a textual\n    description of its parameters. The Parameter spec needs to match\n    the number and names of the callables arguments.\n    \"\"\"\n    from inspect import getargspec\n    # get the signature\n    ndefaults = 0\n    args, varargs, varkw, defaults = getargspec(func)\n    if not defaults is None:\n        ndefaults = len(defaults)\n    # start documentation with what the callable brings with it\n    doc = prefix if prefix else u''\n    if len(args) > 1:\n        if len(doc):\n            doc += '\\n'\n        doc += \"Parameters\\n----------\\n\"\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            # we need a parameter spec for each argument\n            if not arg in params:\n                raise ValueError(\"function has argument '%s' not described as a parameter\" % arg)\n            param = params[arg]\n            # validate the default -- to make sure that the parameter description is\n            # somewhat OK\n            defaults_idx = ndefaults - len(args) + i\n            if defaults_idx >= 0:\n                if not param.constraints is None:\n                    param.constraints(defaults[defaults_idx])\n            doc += param.get_autodoc(\n                arg,\n                default=defaults[defaults_idx] if defaults_idx >= 0 else None,\n                has_default=defaults_idx >= 0)\n            doc += '\\n'\n    doc += suffix if suffix else u\"\"\n    # assign the amended docs\n    func.__doc__ = doc\n    return func\n\n\nclass Interface(object):\n    \"\"\"Base class for interface implementations\"\"\"\n\n    @classmethod\n    def setup_parser(cls, parser):\n        # XXX needs safety check for name collisions\n        # XXX allow for parser kwargs customization\n        parser_kwargs = {}\n        from inspect import getargspec\n        # get the signature\n        ndefaults = 0\n        args, varargs, varkw, defaults = getargspec(cls.__call__)\n        if not defaults is None:\n            ndefaults = len(defaults)\n        for i, arg in enumerate(args):\n            if arg == 'self':\n                continue\n            param = cls._params_[arg]\n            defaults_idx = ndefaults - len(args) + i\n            cmd_args = param.cmd_args\n            if cmd_args is None:\n                cmd_args = []\n            if not len(cmd_args):\n                if defaults_idx >= 0:\n                    # dealing with a kwarg\n                    template = '--%s'\n                else:\n                    # positional arg\n                    template = '%s'\n                # use parameter name as default argument name\n                parser_args = (template % arg.replace('_', '-'),)\n            else:\n                parser_args = [c.replace('_', '-') for c in cmd_args]\n            parser_kwargs = param.cmd_kwargs\n            if defaults_idx >= 0:\n                parser_kwargs['default'] = defaults[defaults_idx]\n            help = alter_interface_docs_for_cmdline(param._doc)\n            if help and help[-1] != '.':\n                help += '.'\n            if param.constraints is not None:\n                parser_kwargs['type'] = param.constraints\n                # include value contraint description and default\n                # into the help string\n                cdoc = alter_interface_docs_for_cmdline(\n                    param.constraints.long_description())\n                if cdoc[0] == '(' and cdoc[-1] == ')':\n                    cdoc = cdoc[1:-1]\n                help += '  Constraints: %s' % cdoc\n            if defaults_idx >= 0:\n                help += \" [Default: %r]\" % (defaults[defaults_idx],)\n            # create the parameter, using the constraint instance for type\n            # conversion\n            parser.add_argument(*parser_args, help=help,\n                                **parser_kwargs)\n\n    @classmethod\n    def call_from_parser(cls, args):\n        # XXX needs safety check for name collisions\n        from inspect import getargspec\n        argnames = getargspec(cls.__call__)[0]\n        kwargs = {k: getattr(args, k) for k in argnames if k != 'self'}\n        try:\n            return cls.__call__(**kwargs)\n        except KeyboardInterrupt:\n            ui.error(\"\\nInterrupted by user while doing magic\")\n            sys.exit(1)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** RST Role Markup Not Stripped in Command Line Documentation

**Description:**
When generating command line documentation, RST role markup such as `:murks:` is not being removed properly. This results in documentation strings retaining unwanted RST syntax, which can lead to rendering issues or confusion.

**Example Buggy Code:**
```python
from datalad.interface.base import alter_interface_docs_for_cmdline

doc_string = ':murks:`me and my buddies`'
processed_doc = alter_interface_docs_for_cmdline(doc_string)
print(processed_doc)
```

**Expected Behavior:**
The `alter_interface_docs_for_cmdline` function should strip out the RST role markup, resulting in the following output:
```
me and my buddies
```

**Actual Behavior:**
The function retains the RST role markup, producing:
```
:murks:`me and my buddies`
```
[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/interface/base.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""High-level interface generation

"""

__docformat__ = 'restructuredtext'

import sys
import re

from ..ui import ui


def get_api_name(intfspec):
    """Given an interface specification return an API name for it"""
    if len(intfspec) > 3:
        name = intfspec[3]
    else:
        name = intfspec[0].split('.')[-1]
    return name


def get_cmdline_command_name(intfspec):
    """Given an interface specification return a cmdline command name"""
    if len(intfspec) > 2:
        name = intfspec[2]
    else:
        name = intfspec[0].split('.')[-1].replace('_', '-')
    return name


def get_interface_groups():
    from .. import interface as _interfaces

    grps = []
    # auto detect all available interfaces and generate a function-based
    # API from them
    for _item in _interfaces.__dict__:
        if not _item.startswith('_group_'):
            continue
        grp_name = _item[7:]
        grp = getattr(_interfaces, _item)
        grps.append((grp_name,) + grp)
    return grps


def dedent_docstring(text):
    import textwrap
    """Remove uniform indentation from a multiline docstring"""
    # Problem is that first line might often have no offset, so might
    # need to be ignored from dedent call
    if text is None:
        return None
    if not text.startswith(' '):
        lines = text.split('\n')
        if len(lines) == 1:
            # single line, no indentation, nothing to do
            return text
        text2 = '\n'.join(lines[1:])
        return lines[0] + "\n" + textwrap.dedent(text2)
    else:
        return textwrap.dedent(text)


def alter_interface_docs_for_api(docs):
    """Apply modifications to interface docstrings for Python API use."""
    # central place to alter the impression of docstrings,
    # like removing cmdline specific sections
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        '\|\| Command line use only \>\>.*\<\< Command line use only \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        '\[CMD:\s.*CMD\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    return docs


def alter_interface_docs_for_cmdline(docs):
    """Apply modifications to interface docstrings for cmdline doc use."""
    # central place to alter the impression of docstrings,
    # like removing Python API specific sections, and argument markup
    if not docs:
        return docs
    docs = dedent_docstring(docs)
    # clean cmdline sections
    docs = re.sub(
        '\|\| Python use only \>\>.*\<\< Python use only \|\|',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # clean cmdline in-line bits
    docs = re.sub(
        '\[PY:\s.*PY\]',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # remove None constraint. In general, `None` on the cmdline means don't
    # give option at all, but specifying `None` explicitly is practically
    # impossible
    docs = re.sub(
        ',\sor\svalue\smust\sbe\s`None`',
        '',
        docs,
        flags=re.MULTILINE | re.DOTALL)
    # capitalize variables and remove backticks to uniformize with
    # argparse output
    docs = re.sub(
        '`\S*`',
        lambda match: match.group(0).strip('`').upper(),
        docs)
    return docs


def update_docstring_with_parameters(func, params, prefix=None, suffix=None):
    """Generate a useful docstring from a parameter spec

    Amends any existing docstring of a callable with a textual
    description of its parameters. The Parameter spec needs to match
    the number and names of the callables arguments.
    """
    from inspect import getargspec
    # get the signature
    ndefaults = 0
    args, varargs, varkw, defaults = getargspec(func)
    if not defaults is None:
        ndefaults = len(defaults)
    # start documentation with what the callable brings with it
    doc = prefix if prefix else u''
    if len(args) > 1:
        if len(doc):
            doc += '\n'
        doc += "Parameters\n----------\n"
        for i, arg in enumerate(args):
            if arg == 'self':
                continue
            # we need a parameter spec for each argument
            if not arg in params:
                raise ValueError("function has argument '%s' not described as a parameter" % arg)
            param = params[arg]
            # validate the default -- to make sure that the parameter description is
            # somewhat OK
            defaults_idx = ndefaults - len(args) + i
            if defaults_idx >= 0:
                if not param.constraints is None:
                    param.constraints(defaults[defaults_idx])
            doc += param.get_autodoc(
                arg,
                default=defaults[defaults_idx] if defaults_idx >= 0 else None,
                has_default=defaults_idx >= 0)
            doc += '\n'
    doc += suffix if suffix else u""
    # assign the amended docs
    func.__doc__ = doc
    return func


class Interface(object):
    """Base class for interface implementations"""

    @classmethod
    def setup_parser(cls, parser):
        # XXX needs safety check for name collisions
        # XXX allow for parser kwargs customization
        parser_kwargs = {}
        from inspect import getargspec
        # get the signature
        ndefaults = 0
        args, varargs, varkw, defaults = getargspec(cls.__call__)
        if not defaults is None:
            ndefaults = len(defaults)
        for i, arg in enumerate(args):
            if arg == 'self':
                continue
            param = cls._params_[arg]
            defaults_idx = ndefaults - len(args) + i
            cmd_args = param.cmd_args
            if cmd_args is None:
                cmd_args = []
            if not len(cmd_args):
                if defaults_idx >= 0:
                    # dealing with a kwarg
                    template = '--%s'
                else:
                    # positional arg
                    template = '%s'
                # use parameter name as default argument name
                parser_args = (template % arg.replace('_', '-'),)
            else:
                parser_args = [c.replace('_', '-') for c in cmd_args]
            parser_kwargs = param.cmd_kwargs
            if defaults_idx >= 0:
                parser_kwargs['default'] = defaults[defaults_idx]
            help = alter_interface_docs_for_cmdline(param._doc)
            if help and help[-1] != '.':
                help += '.'
            if param.constraints is not None:
                parser_kwargs['type'] = param.constraints
                # include value contraint description and default
                # into the help string
                cdoc = alter_interface_docs_for_cmdline(
                    param.constraints.long_description())
                if cdoc[0] == '(' and cdoc[-1] == ')':
                    cdoc = cdoc[1:-1]
                help += '  Constraints: %s' % cdoc
            if defaults_idx >= 0:
                help += " [Default: %r]" % (defaults[defaults_idx],)
            # create the parameter, using the constraint instance for type
            # conversion
            parser.add_argument(*parser_args, help=help,
                                **parser_kwargs)

    @classmethod
    def call_from_parser(cls, args):
        # XXX needs safety check for name collisions
        from inspect import getargspec
        argnames = getargspec(cls.__call__)[0]
        kwargs = {k: getattr(args, k) for k in argnames if k != 'self'}
        try:
            return cls.__call__(**kwargs)
        except KeyboardInterrupt:
            ui.error("\nInterrupted by user while doing magic")
            sys.exit(1)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user