{
    "file_diffs": [
        {
            "old_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"\nWrapper for command and function calls, allowing for dry runs and output handling\n\n\"\"\"\n\n\nimport subprocess\nimport sys\nimport logging\nimport os\nimport shutil\nimport shlex\nimport atexit\nimport functools\nimport tempfile\n\nfrom collections import OrderedDict\nfrom six import PY3, PY2\nfrom six import string_types, binary_type, text_type\nfrom os.path import abspath, isabs, pathsep, exists\n\nfrom .consts import GIT_SSH_COMMAND\nfrom .dochelpers import exc_str\nfrom .support.exceptions import CommandError\nfrom .support.protocol import NullProtocol, DryRunProtocol, \\\n    ExecutionTimeProtocol, ExecutionTimeExternalsProtocol\nfrom .utils import on_windows, get_tempfile_kwargs, assure_unicode\nfrom .dochelpers import borrowdoc\n\nlgr = logging.getLogger('datalad.cmd')\n\n# In python3 to split byte stream on newline, it must be bytes\nlinesep_bytes = os.linesep.encode()\n\n_TEMP_std = sys.stdout, sys.stderr\n# To be used in the temp file name to distinguish the ones we create\n# in Runner so we take care about their removal, in contrast to those\n# which might be created outside and passed into Runner\n_MAGICAL_OUTPUT_MARKER = \"_runneroutput_\"\n\nif PY2:\n    # TODO apparently there is a recommended substitution for Python2\n    # which is a backported implementation of python3 subprocess\n    # https://pypi.python.org/pypi/subprocess32/\n    file_class = file\nelse:\n    from io import IOBase as file_class\n\n\ndef _decide_to_log(v):\n    \"\"\"Hacky workaround for now so we could specify per each which to\n    log online and which to the log\"\"\"\n    if isinstance(v, bool) or callable(v):\n        return v\n    elif v in {'online'}:\n        return True\n    elif v in {'offline'}:\n        return False\n    else:\n        raise ValueError(\"can be bool, callable, 'online' or 'offline'\")\n\n\ndef _get_output_stream(log_std, false_value):\n    \"\"\"Helper to prepare output stream for Popen and use file for 'offline'\n\n    Necessary to avoid lockdowns when both stdout and stderr are pipes\n    \"\"\"\n    if log_std:\n        if log_std == 'offline':\n            # we will open a temporary file\n\n            tf = tempfile.mktemp(\n                **get_tempfile_kwargs({}, prefix=_MAGICAL_OUTPUT_MARKER)\n            )\n            return open(tf, 'w')  # XXX PY3 should be 'b' may be?\n        else:\n            return subprocess.PIPE\n    else:\n        return false_value\n\n\ndef _cleanup_output(stream, std):\n    if isinstance(stream, file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n        if not stream.closed:\n            stream.close()\n        if exists(stream.name):\n            os.unlink(stream.name)\n    elif stream == subprocess.PIPE:\n        std.close()\n\n\nclass Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                for line in out_.split(linesep_bytes):\n                    out += self._process_one_line(*pargs, line=line)\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )\n\n\nclass GitRunner(Runner):\n    \"\"\"\n    Runner to be used to run git and git annex commands\n\n    Overloads the runner class to check & update GIT_DIR and\n    GIT_WORK_TREE environment variables set to the absolute path\n    if is defined and is relative path\n    \"\"\"\n    _GIT_PATH = None\n\n    @borrowdoc(Runner)\n    def __init__(self, *args, **kwargs):\n        super(GitRunner, self).__init__(*args, **kwargs)\n        self._check_git_path()\n\n    @staticmethod\n    def _check_git_path():\n        \"\"\"If using bundled git-annex, we would like to use bundled with it git\n\n        Thus we will store _GIT_PATH a path to git in the same directory as annex\n        if found.  If it is empty (but not None), we do nothing\n        \"\"\"\n        if GitRunner._GIT_PATH is None:\n            from distutils.spawn import find_executable\n            # with all the nesting of config and this runner, cannot use our\n            # cfg here, so will resort to dark magic of environment options\n            if (os.environ.get('DATALAD_USE_DEFAULT_GIT', '0').lower()\n                    in ('1', 'on', 'true', 'yes')):\n                git_fpath = find_executable(\"git\")\n                if git_fpath:\n                    GitRunner._GIT_PATH = ''\n                    lgr.log(9, \"Will use default git %s\", git_fpath)\n                    return  # we are done - there is a default git avail.\n                # if not -- we will look for a bundled one\n\n            annex_fpath = find_executable(\"git-annex\")\n            if not annex_fpath:\n                # not sure how to live further anyways! ;)\n                alongside = False\n            else:\n                annex_path = os.path.dirname(os.path.realpath(annex_fpath))\n                if on_windows:\n                    # just bundled installations so git should be taken from annex\n                    alongside = True\n                else:\n                    alongside = os.path.lexists(os.path.join(annex_path, 'git'))\n            GitRunner._GIT_PATH = annex_path if alongside else ''\n            lgr.log(9, \"Will use git under %r (no adjustments to PATH if empty \"\n                       \"string)\", GitRunner._GIT_PATH)\n            assert(GitRunner._GIT_PATH is not None)  # we made the decision!\n\n    @staticmethod\n    def get_git_environ_adjusted(env=None):\n        \"\"\"\n        Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined\n        \"\"\"\n        # if env set copy else get os environment\n        git_env = env.copy() if env else os.environ.copy()\n        if GitRunner._GIT_PATH:\n            git_env['PATH'] = pathsep.join([GitRunner._GIT_PATH, git_env['PATH']]) \\\n                if 'PATH' in git_env \\\n                else GitRunner._GIT_PATH\n\n        for varstring in ['GIT_DIR', 'GIT_WORK_TREE']:\n            var = git_env.get(varstring)\n            if var:                                    # if env variable set\n                if not isabs(var):                     # and it's a relative path\n                    git_env[varstring] = abspath(var)  # to absolute path\n                    lgr.log(9, \"Updated %s to %s\", varstring, git_env[varstring])\n\n        if 'GIT_SSH_COMMAND' not in git_env:\n            git_env['GIT_SSH_COMMAND'] = GIT_SSH_COMMAND\n\n        return git_env\n\n    def run(self, cmd, env=None, *args, **kwargs):\n        return super(GitRunner, self).run(\n            cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)\n\n\n# ####\n# Preserve from previous version\n# TODO: document intention\n# ####\n# this one might get under Runner for better output/control\ndef link_file_load(src, dst, dry_run=False):\n    \"\"\"Just a little helper to hardlink files's load\n    \"\"\"\n    dst_dir = os.path.dirname(dst)\n    if not os.path.exists(dst_dir):\n        os.makedirs(dst_dir)\n    if os.path.lexists(dst):\n        lgr.log(9, \"Destination file %(dst)s exists. Removing it first\", locals())\n        # TODO: how would it interact with git/git-annex\n        os.unlink(dst)\n    lgr.log(9, \"Hardlinking %(src)s under %(dst)s\", locals())\n    src_realpath = os.path.realpath(src)\n\n    try:\n        os.link(src_realpath, dst)\n    except AttributeError as e:\n        lgr.warn(\"Linking of %s failed (%s), copying file\" % (src, e))\n        shutil.copyfile(src_realpath, dst)\n        shutil.copystat(src_realpath, dst)\n    else:\n        lgr.log(2, \"Hardlinking finished\")\n\n\ndef get_runner(*args, **kwargs):\n    # needs local import, because the ConfigManager itself needs the runner\n    from . import cfg\n    # TODO:  this is all crawl specific -- should be moved away\n    if cfg.obtain('datalad.crawl.dryrun', default=False):\n        kwargs = kwargs.copy()\n        kwargs['protocol'] = DryRunProtocol()\n    return Runner(*args, **kwargs)\n",
            "new_file_content": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"\nWrapper for command and function calls, allowing for dry runs and output handling\n\n\"\"\"\n\n\nimport subprocess\nimport sys\nimport logging\nimport os\nimport shutil\nimport shlex\nimport atexit\nimport functools\nimport tempfile\n\nfrom collections import OrderedDict\nfrom six import PY3, PY2\nfrom six import string_types, binary_type, text_type\nfrom os.path import abspath, isabs, pathsep, exists\n\nfrom .consts import GIT_SSH_COMMAND\nfrom .dochelpers import exc_str\nfrom .support.exceptions import CommandError\nfrom .support.protocol import NullProtocol, DryRunProtocol, \\\n    ExecutionTimeProtocol, ExecutionTimeExternalsProtocol\nfrom .utils import on_windows, get_tempfile_kwargs, assure_unicode\nfrom .dochelpers import borrowdoc\n\nlgr = logging.getLogger('datalad.cmd')\n\n# In python3 to split byte stream on newline, it must be bytes\nlinesep_bytes = os.linesep.encode()\n\n_TEMP_std = sys.stdout, sys.stderr\n# To be used in the temp file name to distinguish the ones we create\n# in Runner so we take care about their removal, in contrast to those\n# which might be created outside and passed into Runner\n_MAGICAL_OUTPUT_MARKER = \"_runneroutput_\"\n\nif PY2:\n    # TODO apparently there is a recommended substitution for Python2\n    # which is a backported implementation of python3 subprocess\n    # https://pypi.python.org/pypi/subprocess32/\n    file_class = file\nelse:\n    from io import IOBase as file_class\n\n\ndef _decide_to_log(v):\n    \"\"\"Hacky workaround for now so we could specify per each which to\n    log online and which to the log\"\"\"\n    if isinstance(v, bool) or callable(v):\n        return v\n    elif v in {'online'}:\n        return True\n    elif v in {'offline'}:\n        return False\n    else:\n        raise ValueError(\"can be bool, callable, 'online' or 'offline'\")\n\n\ndef _get_output_stream(log_std, false_value):\n    \"\"\"Helper to prepare output stream for Popen and use file for 'offline'\n\n    Necessary to avoid lockdowns when both stdout and stderr are pipes\n    \"\"\"\n    if log_std:\n        if log_std == 'offline':\n            # we will open a temporary file\n\n            tf = tempfile.mktemp(\n                **get_tempfile_kwargs({}, prefix=_MAGICAL_OUTPUT_MARKER)\n            )\n            return open(tf, 'w')  # XXX PY3 should be 'b' may be?\n        else:\n            return subprocess.PIPE\n    else:\n        return false_value\n\n\ndef _cleanup_output(stream, std):\n    if isinstance(stream, file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n        if not stream.closed:\n            stream.close()\n        if exists(stream.name):\n            os.unlink(stream.name)\n    elif stream == subprocess.PIPE:\n        std.close()\n\n\nclass Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                out += linesep_bytes.join(\n                    self._process_one_line(*pargs, line=line)\n                    for line in out_.split(linesep_bytes))\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )\n\n\nclass GitRunner(Runner):\n    \"\"\"\n    Runner to be used to run git and git annex commands\n\n    Overloads the runner class to check & update GIT_DIR and\n    GIT_WORK_TREE environment variables set to the absolute path\n    if is defined and is relative path\n    \"\"\"\n    _GIT_PATH = None\n\n    @borrowdoc(Runner)\n    def __init__(self, *args, **kwargs):\n        super(GitRunner, self).__init__(*args, **kwargs)\n        self._check_git_path()\n\n    @staticmethod\n    def _check_git_path():\n        \"\"\"If using bundled git-annex, we would like to use bundled with it git\n\n        Thus we will store _GIT_PATH a path to git in the same directory as annex\n        if found.  If it is empty (but not None), we do nothing\n        \"\"\"\n        if GitRunner._GIT_PATH is None:\n            from distutils.spawn import find_executable\n            # with all the nesting of config and this runner, cannot use our\n            # cfg here, so will resort to dark magic of environment options\n            if (os.environ.get('DATALAD_USE_DEFAULT_GIT', '0').lower()\n                    in ('1', 'on', 'true', 'yes')):\n                git_fpath = find_executable(\"git\")\n                if git_fpath:\n                    GitRunner._GIT_PATH = ''\n                    lgr.log(9, \"Will use default git %s\", git_fpath)\n                    return  # we are done - there is a default git avail.\n                # if not -- we will look for a bundled one\n\n            annex_fpath = find_executable(\"git-annex\")\n            if not annex_fpath:\n                # not sure how to live further anyways! ;)\n                alongside = False\n            else:\n                annex_path = os.path.dirname(os.path.realpath(annex_fpath))\n                if on_windows:\n                    # just bundled installations so git should be taken from annex\n                    alongside = True\n                else:\n                    alongside = os.path.lexists(os.path.join(annex_path, 'git'))\n            GitRunner._GIT_PATH = annex_path if alongside else ''\n            lgr.log(9, \"Will use git under %r (no adjustments to PATH if empty \"\n                       \"string)\", GitRunner._GIT_PATH)\n            assert(GitRunner._GIT_PATH is not None)  # we made the decision!\n\n    @staticmethod\n    def get_git_environ_adjusted(env=None):\n        \"\"\"\n        Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined\n        \"\"\"\n        # if env set copy else get os environment\n        git_env = env.copy() if env else os.environ.copy()\n        if GitRunner._GIT_PATH:\n            git_env['PATH'] = pathsep.join([GitRunner._GIT_PATH, git_env['PATH']]) \\\n                if 'PATH' in git_env \\\n                else GitRunner._GIT_PATH\n\n        for varstring in ['GIT_DIR', 'GIT_WORK_TREE']:\n            var = git_env.get(varstring)\n            if var:                                    # if env variable set\n                if not isabs(var):                     # and it's a relative path\n                    git_env[varstring] = abspath(var)  # to absolute path\n                    lgr.log(9, \"Updated %s to %s\", varstring, git_env[varstring])\n\n        if 'GIT_SSH_COMMAND' not in git_env:\n            git_env['GIT_SSH_COMMAND'] = GIT_SSH_COMMAND\n\n        return git_env\n\n    def run(self, cmd, env=None, *args, **kwargs):\n        return super(GitRunner, self).run(\n            cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)\n\n\n# ####\n# Preserve from previous version\n# TODO: document intention\n# ####\n# this one might get under Runner for better output/control\ndef link_file_load(src, dst, dry_run=False):\n    \"\"\"Just a little helper to hardlink files's load\n    \"\"\"\n    dst_dir = os.path.dirname(dst)\n    if not os.path.exists(dst_dir):\n        os.makedirs(dst_dir)\n    if os.path.lexists(dst):\n        lgr.log(9, \"Destination file %(dst)s exists. Removing it first\", locals())\n        # TODO: how would it interact with git/git-annex\n        os.unlink(dst)\n    lgr.log(9, \"Hardlinking %(src)s under %(dst)s\", locals())\n    src_realpath = os.path.realpath(src)\n\n    try:\n        os.link(src_realpath, dst)\n    except AttributeError as e:\n        lgr.warn(\"Linking of %s failed (%s), copying file\" % (src, e))\n        shutil.copyfile(src_realpath, dst)\n        shutil.copystat(src_realpath, dst)\n    else:\n        lgr.log(2, \"Hardlinking finished\")\n\n\ndef get_runner(*args, **kwargs):\n    # needs local import, because the ConfigManager itself needs the runner\n    from . import cfg\n    # TODO:  this is all crawl specific -- should be moved away\n    if cfg.obtain('datalad.crawl.dryrun', default=False):\n        kwargs = kwargs.copy()\n        kwargs['protocol'] = DryRunProtocol()\n    return Runner(*args, **kwargs)\n",
            "header": {
                "file": {
                    "path": "datalad/cmd.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "be04eb5cd",
                "new_commit_hash": "8d5fb026e",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/cmd.py"
            },
            "plus_file": {
                "path": "b/datalad/cmd.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 326,
                            "length": 8
                        },
                        "new_range": {
                            "start": 326,
                            "length": 9
                        },
                        "section": "class Runner(object):"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "                # resolving a once in a while failing test #2185",
                                "type": "context"
                            },
                            {
                                "content": "                if isinstance(out_, text_type):",
                                "type": "context"
                            },
                            {
                                "content": "                    out_ = out_.encode('utf-8')",
                                "type": "context"
                            },
                            {
                                "content": "                for line in out_.split(linesep_bytes):",
                                "type": "deleted"
                            },
                            {
                                "content": "                    out += self._process_one_line(*pargs, line=line)",
                                "type": "deleted"
                            },
                            {
                                "content": "                out += linesep_bytes.join(",
                                "type": "added"
                            },
                            {
                                "content": "                    self._process_one_line(*pargs, line=line)",
                                "type": "added"
                            },
                            {
                                "content": "                    for line in out_.split(linesep_bytes))",
                                "type": "added"
                            },
                            {
                                "content": "        return out",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            },
                            {
                                "content": "    def _process_one_line(self, out_type, proc, log_, log_is_callable,",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/cmd.py",
                            "type": "method",
                            "name": "Runner._process_remaining_output",
                            "content": "class Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                for line in out_.split(linesep_bytes):\n                    out += self._process_one_line(*pargs, line=line)\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 312,
                            "end_lineno": 331,
                            "parent": {
                                "file_name": "datalad/cmd.py",
                                "type": "class",
                                "name": "Runner",
                                "content": "class Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                for line in out_.split(linesep_bytes):\n                    out += self._process_one_line(*pargs, line=line)\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )",
                                "ast_type_str": "ClassDef",
                                "start_lineno": 100,
                                "end_lineno": 584,
                                "parent": null
                            }
                        },
                        {
                            "file_name": "datalad/cmd.py",
                            "type": "class",
                            "name": "Runner",
                            "content": "class Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                for line in out_.split(linesep_bytes):\n                    out += self._process_one_line(*pargs, line=line)\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )",
                            "ast_type_str": "ClassDef",
                            "start_lineno": 100,
                            "end_lineno": 584,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        },
        {
            "old_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test command call wrapper\n\"\"\"\n\nfrom mock import patch\nimport os\nfrom os.path import dirname, join as opj\nimport sys\nimport logging\nimport shlex\n\nfrom .utils import ok_, eq_, assert_is, assert_equal, assert_false, \\\n    assert_true, assert_greater, assert_raises, assert_in, SkipTest\n\nfrom ..cmd import Runner, link_file_load\nfrom ..cmd import GitRunner\nfrom ..support.exceptions import CommandError\nfrom ..support.protocol import DryRunProtocol\nfrom .utils import with_tempfile, assert_cwd_unchanged, \\\n    ignore_nose_capturing_stdout, swallow_outputs, swallow_logs, \\\n    on_linux, on_osx, on_windows, with_testrepos\nfrom .utils import lgr\nfrom ..utils import assure_unicode\n\nfrom .utils import local_testrepo_flavors\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner_dry(tempfile):\n\n    dry = DryRunProtocol()\n    runner = Runner(protocol=dry)\n\n    # test dry command call\n    cmd = 'echo Testing dry run > %s' % tempfile\n    with swallow_logs(new_level=9) as cml:\n        ret = runner.run(cmd)\n        cml.assert_logged(\"{DryRunProtocol} Running: %s\" % cmd, regex=False)\n    assert_equal((\"DRY\", \"DRY\"), ret,\n                 \"Output of dry run (%s): %s\" % (cmd, ret))\n    assert_equal(shlex.split(cmd, posix=not on_windows), dry[0]['command'])\n    assert_false(os.path.exists(tempfile))\n\n    # test dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_is(None, output, \"Dry call of: os.path.join, 'foo', 'bar' \"\n                            \"returned: %s\" % output)\n    assert_in('join', dry[1]['command'][0])\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1])\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner(tempfile):\n\n    # test non-dry command call\n    runner = Runner()\n    cmd = 'echo Testing real run > %r' % tempfile\n    ret = runner.run(cmd)\n    assert_true(os.path.exists(tempfile),\n                \"Run of: %s resulted with non-existing file %s\" %\n                (cmd, tempfile))\n\n    # test non-dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_equal(os.path.join('foo', 'bar'), output,\n                 \"Call of: os.path.join, 'foo', 'bar' returned %s\" % output)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_dry():\n\n    cmd_ = ['echo', 'Testing', '__call__', 'with', 'string']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        dry = DryRunProtocol()\n        runner = Runner(protocol=dry)\n        ret = runner(cmd)\n        # (stdout, stderr) is returned.  But in dry -- (\"DRY\",\"DRY\")\n        eq_(ret, (\"DRY\", \"DRY\"))\n        assert_equal(cmd_, dry[0]['command'],\n                     \"Dry run of Runner.__call__ didn't record command: %s.\\n\"\n                     \"Buffer: %s\" % (cmd, dry))\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, None)\n\n    assert_in('join', dry[1]['command'][0],\n              \"Dry run of Runner.__call__ didn't record function join().\"\n              \"Buffer: %s\" % dry)\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1],\n                 \"Dry run of Runner.__call__ didn't record function join().\"\n                 \"Buffer: %s\" % dry)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_wet():\n\n    runner = Runner()\n    cmd = [sys.executable, \"-c\", \"print('Testing')\"]\n\n    out = runner(cmd)\n    eq_(out[0].rstrip(), ('Testing'))\n    eq_(out[1], '')\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, os.path.join('foo', 'bar'))\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stderr():\n\n    runner = Runner(log_outputs=True)\n    cmd = 'echo stderr-Message should be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=True, expect_stderr=True)\n            cml.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cml.assert_logged(\"stderr| stderr-\"\n                                  \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cml.lines[1])\n\n    cmd = 'echo stderr-Message should not be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=False)\n            eq_(cmo.err.rstrip(), \"stderr-Message should not be logged\")\n            assert_raises(AssertionError, cml.assert_logged,\n                          \"stderr| stderr-Message should not be logged\")\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stdout():\n    # TODO: no idea of how to check correct logging via any kind of\n    # assertion yet.\n\n    runner = Runner(log_outputs=True)\n    cmd_ = ['echo', 'stdout-Message should be logged']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        # should be identical runs, either as a string or as a list\n        kw = {}\n        # on Windows it can't find echo if ran outside the shell\n        if on_windows and isinstance(cmd, list):\n            kw['shell'] = True\n        with swallow_logs(9) as cm:\n            ret = runner.run(cmd, log_stdout=True, **kw)\n            cm.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cm.assert_logged(\"stdout| stdout-\"\n                                 \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cm.lines[1])\n\n    cmd = 'echo stdout-Message should not be logged'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=11) as cml:\n            ret = runner.run(cmd, log_stdout=False)\n            eq_(cmo.out, \"stdout-Message should not be logged\\n\")\n            eq_(cml.out, \"\")\n\n\n@ignore_nose_capturing_stdout\ndef check_runner_heavy_output(log_online):\n    # TODO: again, no automatic detection of this resulting in being\n    # stucked yet.\n\n    runner = Runner()\n    cmd = '%s %s' % (sys.executable, opj(dirname(__file__), \"heavyoutput.py\"))\n\n    with swallow_outputs() as cm, swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=log_online,\n                         log_stderr=False, log_stdout=False,\n                         expect_stderr=True)\n        eq_(cm.err, cm.out)  # they are identical in that script\n        eq_(cm.out[:10], \"0 [0, 1, 2\")\n        eq_(cm.out[-15:], \"997, 998, 999]\\n\")\n\n    # for some reason swallow_logs is not effective, so we just skip altogether\n    # if too heavy debug output\n    if lgr.getEffectiveLevel() <= logging.DEBUG:\n        raise SkipTest(\"Skipping due to too heavy impact on logs complicating debugging\")\n\n    #do it again with capturing:\n    with swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=True, log_stderr=True, log_stdout=True,\n                         expect_stderr=True)\n\n    if log_online:\n        # halting case of datalad add and other batch commands #2116\n        logged = []\n        with swallow_logs():\n            def process_stdout(l):\n                assert l\n                logged.append(l)\n            ret = runner.run(\n                cmd,\n                log_online=log_online,\n                log_stdout=process_stdout,\n                log_stderr='offline',\n                expect_stderr=True\n            )\n        assert_equal(len(logged), 100)\n        assert_greater(len(ret[1]), 1000)  # stderr all here\n        assert not ret[0], \"all messages went into `logged`\"\n\n\ndef test_runner_heavy_output():\n    for log_online in [False, True]:\n        yield check_runner_heavy_output, log_online\n\n\n@with_tempfile\ndef test_link_file_load(tempfile):\n    tempfile2 = tempfile + '_'\n\n    with open(tempfile, 'w') as f:\n        f.write(\"LOAD\")\n\n    link_file_load(tempfile, tempfile2)  # this should work in general\n\n    ok_(os.path.exists(tempfile2))\n\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n\n    def inode(fname):\n        with open(fname) as fd:\n            return os.fstat(fd.fileno()).st_ino\n\n    def stats(fname, times=True):\n        \"\"\"Return stats on the file which should have been preserved\"\"\"\n        with open(fname) as fd:\n            st = os.fstat(fd.fileno())\n            stats = (st.st_mode, st.st_uid, st.st_gid, st.st_size)\n            if times:\n                return stats + (st.st_atime, st.st_mtime)\n            else:\n                return stats\n            # despite copystat mtime is not copied. TODO\n            #        st.st_mtime)\n\n    if on_linux or on_osx:\n        # above call should result in the hardlink\n        assert_equal(inode(tempfile), inode(tempfile2))\n        assert_equal(stats(tempfile), stats(tempfile2))\n\n        # and if we mock absence of .link\n        def raise_AttributeError(*args):\n            raise AttributeError(\"TEST\")\n\n        with patch('os.link', raise_AttributeError):\n            with swallow_logs(logging.WARNING) as cm:\n                link_file_load(tempfile, tempfile2)  # should still work\n                ok_(\"failed (TEST), copying file\" in cm.out)\n\n    # should be a copy (either originally for windows, or after mocked call)\n    ok_(inode(tempfile) != inode(tempfile2))\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n    assert_equal(stats(tempfile, times=False), stats(tempfile2, times=False))\n    os.unlink(tempfile2)  # TODO: next two with_tempfile\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_failure(dir_):\n    from ..support.annexrepo import AnnexRepo\n    repo = AnnexRepo(dir_, create=True)\n    runner = Runner()\n    failing_cmd = ['git-annex', 'add', 'notexistent.dat']\n\n    with assert_raises(CommandError) as cme, \\\n         swallow_logs() as cml:\n        runner.run(failing_cmd, cwd=dir_)\n        assert_in('notexistent.dat not found', cml.out)\n    assert_equal(1, cme.exception.code)\n\n\n@with_tempfile(mkdir=True)\ndef test_git_path(dir_):\n    from ..support.gitrepo import GitRepo\n    # As soon as we use any GitRepo we should get _GIT_PATH set in the Runner\n    repo = GitRepo(dir_, create=True)\n    assert GitRunner._GIT_PATH is not None\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_stdin(path):\n    runner = Runner()\n    with open(opj(path, \"test_input.txt\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    with swallow_outputs() as cmo, open(opj(path, \"test_input.txt\"), \"r\") as fake_input:\n        runner.run(['cat'], log_stdout=False, stdin=fake_input)\n        assert_in(\"whatever\", cmo.out)\n\n\ndef test_process_remaining_output():\n    runner = Runner()\n    out = u\"\"\"\\\ns\nп\n\"\"\"\n    out_bytes = out.encode('utf-8')\n    target = u\"sп\".encode('utf-8')\n    args = ['stdout', None, False, False]\n    #  probably #2185\n    eq_(runner._process_remaining_output(None, out_bytes, *args), target)\n    eq_(runner._process_remaining_output(None, out, *args), target)",
            "new_file_content": "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test command call wrapper\n\"\"\"\n\nfrom mock import patch\nimport os\nfrom os.path import dirname, join as opj\nimport sys\nimport logging\nimport shlex\n\nfrom .utils import ok_, eq_, assert_is, assert_equal, assert_false, \\\n    assert_true, assert_greater, assert_raises, assert_in, SkipTest\n\nfrom ..cmd import Runner, link_file_load\nfrom ..cmd import GitRunner\nfrom ..support.exceptions import CommandError\nfrom ..support.protocol import DryRunProtocol\nfrom .utils import with_tempfile, assert_cwd_unchanged, \\\n    ignore_nose_capturing_stdout, swallow_outputs, swallow_logs, \\\n    on_linux, on_osx, on_windows, with_testrepos\nfrom .utils import lgr\nfrom ..utils import assure_unicode\n\nfrom .utils import local_testrepo_flavors\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner_dry(tempfile):\n\n    dry = DryRunProtocol()\n    runner = Runner(protocol=dry)\n\n    # test dry command call\n    cmd = 'echo Testing dry run > %s' % tempfile\n    with swallow_logs(new_level=9) as cml:\n        ret = runner.run(cmd)\n        cml.assert_logged(\"{DryRunProtocol} Running: %s\" % cmd, regex=False)\n    assert_equal((\"DRY\", \"DRY\"), ret,\n                 \"Output of dry run (%s): %s\" % (cmd, ret))\n    assert_equal(shlex.split(cmd, posix=not on_windows), dry[0]['command'])\n    assert_false(os.path.exists(tempfile))\n\n    # test dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_is(None, output, \"Dry call of: os.path.join, 'foo', 'bar' \"\n                            \"returned: %s\" % output)\n    assert_in('join', dry[1]['command'][0])\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1])\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner(tempfile):\n\n    # test non-dry command call\n    runner = Runner()\n    cmd = 'echo Testing real run > %r' % tempfile\n    ret = runner.run(cmd)\n    assert_true(os.path.exists(tempfile),\n                \"Run of: %s resulted with non-existing file %s\" %\n                (cmd, tempfile))\n\n    # test non-dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_equal(os.path.join('foo', 'bar'), output,\n                 \"Call of: os.path.join, 'foo', 'bar' returned %s\" % output)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_dry():\n\n    cmd_ = ['echo', 'Testing', '__call__', 'with', 'string']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        dry = DryRunProtocol()\n        runner = Runner(protocol=dry)\n        ret = runner(cmd)\n        # (stdout, stderr) is returned.  But in dry -- (\"DRY\",\"DRY\")\n        eq_(ret, (\"DRY\", \"DRY\"))\n        assert_equal(cmd_, dry[0]['command'],\n                     \"Dry run of Runner.__call__ didn't record command: %s.\\n\"\n                     \"Buffer: %s\" % (cmd, dry))\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, None)\n\n    assert_in('join', dry[1]['command'][0],\n              \"Dry run of Runner.__call__ didn't record function join().\"\n              \"Buffer: %s\" % dry)\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1],\n                 \"Dry run of Runner.__call__ didn't record function join().\"\n                 \"Buffer: %s\" % dry)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_wet():\n\n    runner = Runner()\n    cmd = [sys.executable, \"-c\", \"print('Testing')\"]\n\n    out = runner(cmd)\n    eq_(out[0].rstrip(), ('Testing'))\n    eq_(out[1], '')\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, os.path.join('foo', 'bar'))\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stderr():\n\n    runner = Runner(log_outputs=True)\n    cmd = 'echo stderr-Message should be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=True, expect_stderr=True)\n            cml.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cml.assert_logged(\"stderr| stderr-\"\n                                  \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cml.lines[1])\n\n    cmd = 'echo stderr-Message should not be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=False)\n            eq_(cmo.err.rstrip(), \"stderr-Message should not be logged\")\n            assert_raises(AssertionError, cml.assert_logged,\n                          \"stderr| stderr-Message should not be logged\")\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stdout():\n    # TODO: no idea of how to check correct logging via any kind of\n    # assertion yet.\n\n    runner = Runner(log_outputs=True)\n    cmd_ = ['echo', 'stdout-Message should be logged']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        # should be identical runs, either as a string or as a list\n        kw = {}\n        # on Windows it can't find echo if ran outside the shell\n        if on_windows and isinstance(cmd, list):\n            kw['shell'] = True\n        with swallow_logs(9) as cm:\n            ret = runner.run(cmd, log_stdout=True, **kw)\n            cm.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cm.assert_logged(\"stdout| stdout-\"\n                                 \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cm.lines[1])\n\n    cmd = 'echo stdout-Message should not be logged'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=11) as cml:\n            ret = runner.run(cmd, log_stdout=False)\n            eq_(cmo.out, \"stdout-Message should not be logged\\n\")\n            eq_(cml.out, \"\")\n\n\n@ignore_nose_capturing_stdout\ndef check_runner_heavy_output(log_online):\n    # TODO: again, no automatic detection of this resulting in being\n    # stucked yet.\n\n    runner = Runner()\n    cmd = '%s %s' % (sys.executable, opj(dirname(__file__), \"heavyoutput.py\"))\n\n    with swallow_outputs() as cm, swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=log_online,\n                         log_stderr=False, log_stdout=False,\n                         expect_stderr=True)\n        eq_(cm.err, cm.out)  # they are identical in that script\n        eq_(cm.out[:10], \"0 [0, 1, 2\")\n        eq_(cm.out[-15:], \"997, 998, 999]\\n\")\n\n    # for some reason swallow_logs is not effective, so we just skip altogether\n    # if too heavy debug output\n    if lgr.getEffectiveLevel() <= logging.DEBUG:\n        raise SkipTest(\"Skipping due to too heavy impact on logs complicating debugging\")\n\n    #do it again with capturing:\n    with swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=True, log_stderr=True, log_stdout=True,\n                         expect_stderr=True)\n\n    if log_online:\n        # halting case of datalad add and other batch commands #2116\n        logged = []\n        with swallow_logs():\n            def process_stdout(l):\n                assert l\n                logged.append(l)\n            ret = runner.run(\n                cmd,\n                log_online=log_online,\n                log_stdout=process_stdout,\n                log_stderr='offline',\n                expect_stderr=True\n            )\n        assert_equal(len(logged), 100)\n        assert_greater(len(ret[1]), 1000)  # stderr all here\n        assert not ret[0], \"all messages went into `logged`\"\n\n\ndef test_runner_heavy_output():\n    for log_online in [False, True]:\n        yield check_runner_heavy_output, log_online\n\n\n@with_tempfile\ndef test_link_file_load(tempfile):\n    tempfile2 = tempfile + '_'\n\n    with open(tempfile, 'w') as f:\n        f.write(\"LOAD\")\n\n    link_file_load(tempfile, tempfile2)  # this should work in general\n\n    ok_(os.path.exists(tempfile2))\n\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n\n    def inode(fname):\n        with open(fname) as fd:\n            return os.fstat(fd.fileno()).st_ino\n\n    def stats(fname, times=True):\n        \"\"\"Return stats on the file which should have been preserved\"\"\"\n        with open(fname) as fd:\n            st = os.fstat(fd.fileno())\n            stats = (st.st_mode, st.st_uid, st.st_gid, st.st_size)\n            if times:\n                return stats + (st.st_atime, st.st_mtime)\n            else:\n                return stats\n            # despite copystat mtime is not copied. TODO\n            #        st.st_mtime)\n\n    if on_linux or on_osx:\n        # above call should result in the hardlink\n        assert_equal(inode(tempfile), inode(tempfile2))\n        assert_equal(stats(tempfile), stats(tempfile2))\n\n        # and if we mock absence of .link\n        def raise_AttributeError(*args):\n            raise AttributeError(\"TEST\")\n\n        with patch('os.link', raise_AttributeError):\n            with swallow_logs(logging.WARNING) as cm:\n                link_file_load(tempfile, tempfile2)  # should still work\n                ok_(\"failed (TEST), copying file\" in cm.out)\n\n    # should be a copy (either originally for windows, or after mocked call)\n    ok_(inode(tempfile) != inode(tempfile2))\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n    assert_equal(stats(tempfile, times=False), stats(tempfile2, times=False))\n    os.unlink(tempfile2)  # TODO: next two with_tempfile\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_failure(dir_):\n    from ..support.annexrepo import AnnexRepo\n    repo = AnnexRepo(dir_, create=True)\n    runner = Runner()\n    failing_cmd = ['git-annex', 'add', 'notexistent.dat']\n\n    with assert_raises(CommandError) as cme, \\\n         swallow_logs() as cml:\n        runner.run(failing_cmd, cwd=dir_)\n        assert_in('notexistent.dat not found', cml.out)\n    assert_equal(1, cme.exception.code)\n\n\n@with_tempfile(mkdir=True)\ndef test_git_path(dir_):\n    from ..support.gitrepo import GitRepo\n    # As soon as we use any GitRepo we should get _GIT_PATH set in the Runner\n    repo = GitRepo(dir_, create=True)\n    assert GitRunner._GIT_PATH is not None\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_stdin(path):\n    runner = Runner()\n    with open(opj(path, \"test_input.txt\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    with swallow_outputs() as cmo, open(opj(path, \"test_input.txt\"), \"r\") as fake_input:\n        runner.run(['cat'], log_stdout=False, stdin=fake_input)\n        assert_in(\"whatever\", cmo.out)\n\n\ndef test_process_remaining_output():\n    runner = Runner()\n    out = u\"\"\"\\\ns\nп\n\"\"\"\n    out_bytes = out.encode('utf-8')\n    target = u\"s\\nп\\n\".encode('utf-8')\n    args = ['stdout', None, False, False]\n    #  probably #2185\n    eq_(runner._process_remaining_output(None, out_bytes, *args), target)\n    eq_(runner._process_remaining_output(None, out, *args), target)",
            "header": {
                "file": {
                    "path": "datalad/tests/test_cmd.py"
                },
                "misc_line": null
            },
            "index_line": {
                "old_commit_hash": "dbb6a616c",
                "new_commit_hash": "844aaf289",
                "mode": "100644"
            },
            "is_binary_file": false,
            "binary_line": null,
            "minus_file": {
                "path": "a/datalad/tests/test_cmd.py"
            },
            "plus_file": {
                "path": "b/datalad/tests/test_cmd.py"
            },
            "hunks": [
                {
                    "descriptor": {
                        "old_range": {
                            "start": 318,
                            "length": 7
                        },
                        "new_range": {
                            "start": 318,
                            "length": 7
                        },
                        "section": "s"
                    },
                    "line_group": {
                        "all_lines": [
                            {
                                "content": "п",
                                "type": "context"
                            },
                            {
                                "content": "\"\"\"",
                                "type": "context"
                            },
                            {
                                "content": "    out_bytes = out.encode('utf-8')",
                                "type": "context"
                            },
                            {
                                "content": "    target = u\"sп\".encode('utf-8')",
                                "type": "deleted"
                            },
                            {
                                "content": "    target = u\"s\\nп\\n\".encode('utf-8')",
                                "type": "added"
                            },
                            {
                                "content": "    args = ['stdout', None, False, False]",
                                "type": "context"
                            },
                            {
                                "content": "    #  probably #2185",
                                "type": "context"
                            },
                            {
                                "content": "    eq_(runner._process_remaining_output(None, out_bytes, *args), target)",
                                "type": "context"
                            },
                            {
                                "content": "",
                                "type": "context"
                            }
                        ]
                    },
                    "modified_entities": [
                        {
                            "file_name": "datalad/tests/test_cmd.py",
                            "type": "function",
                            "name": "test_process_remaining_output",
                            "content": "def test_process_remaining_output():\n    runner = Runner()\n    out = u\"\"\"\\\ns\nп\n\"\"\"\n    out_bytes = out.encode('utf-8')\n    target = u\"sп\".encode('utf-8')\n    args = ['stdout', None, False, False]\n    #  probably #2185\n    eq_(runner._process_remaining_output(None, out_bytes, *args), target)\n    eq_(runner._process_remaining_output(None, out, *args), target)",
                            "ast_type_str": "FunctionDef",
                            "start_lineno": 314,
                            "end_lineno": 325,
                            "parent": null
                        }
                    ],
                    "added_entities": [],
                    "deleted_entities": []
                }
            ]
        }
    ],
    "old_commit_hash": "6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e^",
    "new_commit_hash": "6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e",
    "commit_message": "Merge pull request #2376 from yarikoptic/bf-2301-missingnewline\n\nBF: assure that each line returned during processing of outputs ends with \\n",
    "commit_date": "2018-04-05T10:59:15+02:00",
    "metadata": {}
}
{
    "repo_name": "datalad",
    "new_commit_hash": "6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e",
    "test_file_codes": [
        "# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil; coding: utf-8 -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"Test command call wrapper\n\"\"\"\n\nfrom mock import patch\nimport os\nfrom os.path import dirname, join as opj\nimport sys\nimport logging\nimport shlex\n\nfrom datalad.tests.utils import ok_, eq_, assert_is, assert_equal, assert_false, \\\n    assert_true, assert_greater, assert_raises, assert_in, SkipTest\n\nfrom datalad.cmd import Runner, link_file_load\nfrom datalad.cmd import GitRunner\nfrom datalad.support.exceptions import CommandError\nfrom datalad.support.protocol import DryRunProtocol\nfrom datalad.tests.utils import with_tempfile, assert_cwd_unchanged, \\\n    ignore_nose_capturing_stdout, swallow_outputs, swallow_logs, \\\n    on_linux, on_osx, on_windows, with_testrepos\nfrom datalad.tests.utils import lgr\nfrom datalad.utils import assure_unicode\n\nfrom datalad.tests.utils import local_testrepo_flavors\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner_dry(tempfile):\n\n    dry = DryRunProtocol()\n    runner = Runner(protocol=dry)\n\n    # test dry command call\n    cmd = 'echo Testing dry run > %s' % tempfile\n    with swallow_logs(new_level=9) as cml:\n        ret = runner.run(cmd)\n        cml.assert_logged(\"{DryRunProtocol} Running: %s\" % cmd, regex=False)\n    assert_equal((\"DRY\", \"DRY\"), ret,\n                 \"Output of dry run (%s): %s\" % (cmd, ret))\n    assert_equal(shlex.split(cmd, posix=not on_windows), dry[0]['command'])\n    assert_false(os.path.exists(tempfile))\n\n    # test dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_is(None, output, \"Dry call of: os.path.join, 'foo', 'bar' \"\n                            \"returned: %s\" % output)\n    assert_in('join', dry[1]['command'][0])\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1])\n\n\n@ignore_nose_capturing_stdout\n@assert_cwd_unchanged\n@with_tempfile\ndef test_runner(tempfile):\n\n    # test non-dry command call\n    runner = Runner()\n    cmd = 'echo Testing real run > %r' % tempfile\n    ret = runner.run(cmd)\n    assert_true(os.path.exists(tempfile),\n                \"Run of: %s resulted with non-existing file %s\" %\n                (cmd, tempfile))\n\n    # test non-dry python function call\n    output = runner.call(os.path.join, 'foo', 'bar')\n    assert_equal(os.path.join('foo', 'bar'), output,\n                 \"Call of: os.path.join, 'foo', 'bar' returned %s\" % output)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_dry():\n\n    cmd_ = ['echo', 'Testing', '__call__', 'with', 'string']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        dry = DryRunProtocol()\n        runner = Runner(protocol=dry)\n        ret = runner(cmd)\n        # (stdout, stderr) is returned.  But in dry -- (\"DRY\",\"DRY\")\n        eq_(ret, (\"DRY\", \"DRY\"))\n        assert_equal(cmd_, dry[0]['command'],\n                     \"Dry run of Runner.__call__ didn't record command: %s.\\n\"\n                     \"Buffer: %s\" % (cmd, dry))\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, None)\n\n    assert_in('join', dry[1]['command'][0],\n              \"Dry run of Runner.__call__ didn't record function join().\"\n              \"Buffer: %s\" % dry)\n    assert_equal(\"args=('foo', 'bar')\", dry[1]['command'][1],\n                 \"Dry run of Runner.__call__ didn't record function join().\"\n                 \"Buffer: %s\" % dry)\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_instance_callable_wet():\n\n    runner = Runner()\n    cmd = [sys.executable, \"-c\", \"print('Testing')\"]\n\n    out = runner(cmd)\n    eq_(out[0].rstrip(), ('Testing'))\n    eq_(out[1], '')\n\n    ret = runner(os.path.join, 'foo', 'bar')\n    eq_(ret, os.path.join('foo', 'bar'))\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stderr():\n\n    runner = Runner(log_outputs=True)\n    cmd = 'echo stderr-Message should be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=True, expect_stderr=True)\n            cml.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cml.assert_logged(\"stderr| stderr-\"\n                                  \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cml.lines[1])\n\n    cmd = 'echo stderr-Message should not be logged >&2'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=9) as cml:\n            ret = runner.run(cmd, log_stderr=False)\n            eq_(cmo.err.rstrip(), \"stderr-Message should not be logged\")\n            assert_raises(AssertionError, cml.assert_logged,\n                          \"stderr| stderr-Message should not be logged\")\n\n\n@ignore_nose_capturing_stdout\ndef test_runner_log_stdout():\n    # TODO: no idea of how to check correct logging via any kind of\n    # assertion yet.\n\n    runner = Runner(log_outputs=True)\n    cmd_ = ['echo', 'stdout-Message should be logged']\n    for cmd in [cmd_, ' '.join(cmd_)]:\n        # should be identical runs, either as a string or as a list\n        kw = {}\n        # on Windows it can't find echo if ran outside the shell\n        if on_windows and isinstance(cmd, list):\n            kw['shell'] = True\n        with swallow_logs(9) as cm:\n            ret = runner.run(cmd, log_stdout=True, **kw)\n            cm.assert_logged(\"Running: %s\" % cmd, level='Level 9', regex=False)\n            if not on_windows:\n                # we can just count on sanity\n                cm.assert_logged(\"stdout| stdout-\"\n                                 \"Message should be logged\", regex=False)\n            else:\n                # echo outputs quoted lines for some reason, so relax check\n                ok_(\"stdout-Message should be logged\" in cm.lines[1])\n\n    cmd = 'echo stdout-Message should not be logged'\n    with swallow_outputs() as cmo:\n        with swallow_logs(new_level=11) as cml:\n            ret = runner.run(cmd, log_stdout=False)\n            eq_(cmo.out, \"stdout-Message should not be logged\\n\")\n            eq_(cml.out, \"\")\n\n\n@ignore_nose_capturing_stdout\ndef check_runner_heavy_output(log_online):\n    # TODO: again, no automatic detection of this resulting in being\n    # stucked yet.\n\n    runner = Runner()\n    cmd = '%s %s' % (sys.executable, opj(dirname(__file__), \"heavyoutput.py\"))\n\n    with swallow_outputs() as cm, swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=log_online,\n                         log_stderr=False, log_stdout=False,\n                         expect_stderr=True)\n        eq_(cm.err, cm.out)  # they are identical in that script\n        eq_(cm.out[:10], \"0 [0, 1, 2\")\n        eq_(cm.out[-15:], \"997, 998, 999]\\n\")\n\n    # for some reason swallow_logs is not effective, so we just skip altogether\n    # if too heavy debug output\n    if lgr.getEffectiveLevel() <= logging.DEBUG:\n        raise SkipTest(\"Skipping due to too heavy impact on logs complicating debugging\")\n\n    #do it again with capturing:\n    with swallow_logs():\n        ret = runner.run(cmd,\n                         log_online=True, log_stderr=True, log_stdout=True,\n                         expect_stderr=True)\n\n    if log_online:\n        # halting case of datalad add and other batch commands #2116\n        logged = []\n        with swallow_logs():\n            def process_stdout(l):\n                assert l\n                logged.append(l)\n            ret = runner.run(\n                cmd,\n                log_online=log_online,\n                log_stdout=process_stdout,\n                log_stderr='offline',\n                expect_stderr=True\n            )\n        assert_equal(len(logged), 100)\n        assert_greater(len(ret[1]), 1000)  # stderr all here\n        assert not ret[0], \"all messages went into `logged`\"\n\n\ndef test_runner_heavy_output():\n    for log_online in [False, True]:\n        yield check_runner_heavy_output, log_online\n\n\n@with_tempfile\ndef test_link_file_load(tempfile):\n    tempfile2 = tempfile + '_'\n\n    with open(tempfile, 'w') as f:\n        f.write(\"LOAD\")\n\n    link_file_load(tempfile, tempfile2)  # this should work in general\n\n    ok_(os.path.exists(tempfile2))\n\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n\n    def inode(fname):\n        with open(fname) as fd:\n            return os.fstat(fd.fileno()).st_ino\n\n    def stats(fname, times=True):\n        \"\"\"Return stats on the file which should have been preserved\"\"\"\n        with open(fname) as fd:\n            st = os.fstat(fd.fileno())\n            stats = (st.st_mode, st.st_uid, st.st_gid, st.st_size)\n            if times:\n                return stats + (st.st_atime, st.st_mtime)\n            else:\n                return stats\n            # despite copystat mtime is not copied. TODO\n            #        st.st_mtime)\n\n    if on_linux or on_osx:\n        # above call should result in the hardlink\n        assert_equal(inode(tempfile), inode(tempfile2))\n        assert_equal(stats(tempfile), stats(tempfile2))\n\n        # and if we mock absence of .link\n        def raise_AttributeError(*args):\n            raise AttributeError(\"TEST\")\n\n        with patch('os.link', raise_AttributeError):\n            with swallow_logs(logging.WARNING) as cm:\n                link_file_load(tempfile, tempfile2)  # should still work\n                ok_(\"failed (TEST), copying file\" in cm.out)\n\n    # should be a copy (either originally for windows, or after mocked call)\n    ok_(inode(tempfile) != inode(tempfile2))\n    with open(tempfile2, 'r') as f:\n        assert_equal(f.read(), \"LOAD\")\n    assert_equal(stats(tempfile, times=False), stats(tempfile2, times=False))\n    os.unlink(tempfile2)  # TODO: next two with_tempfile\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_failure(dir_):\n    from datalad.support.annexrepo import AnnexRepo\n    repo = AnnexRepo(dir_, create=True)\n    runner = Runner()\n    failing_cmd = ['git-annex', 'add', 'notexistent.dat']\n\n    with assert_raises(CommandError) as cme, \\\n         swallow_logs() as cml:\n        runner.run(failing_cmd, cwd=dir_)\n        assert_in('notexistent.dat not found', cml.out)\n    assert_equal(1, cme.exception.code)\n\n\n@with_tempfile(mkdir=True)\ndef test_git_path(dir_):\n    from datalad.support.gitrepo import GitRepo\n    # As soon as we use any GitRepo we should get _GIT_PATH set in the Runner\n    repo = GitRepo(dir_, create=True)\n    assert GitRunner._GIT_PATH is not None\n\n\n@with_tempfile(mkdir=True)\ndef test_runner_stdin(path):\n    runner = Runner()\n    with open(opj(path, \"test_input.txt\"), \"w\") as f:\n        f.write(\"whatever\")\n\n    with swallow_outputs() as cmo, open(opj(path, \"test_input.txt\"), \"r\") as fake_input:\n        runner.run(['cat'], log_stdout=False, stdin=fake_input)\n        assert_in(\"whatever\", cmo.out)\n\n\ndef test_process_remaining_output():\n    runner = Runner()\n    out = u\"\"\"\\\ns\nп\n\"\"\"\n    out_bytes = out.encode('utf-8')\n    target = u\"s\\nп\\n\".encode('utf-8')\n    args = ['stdout', None, False, False]\n    #  probably #2185\n    eq_(runner._process_remaining_output(None, out_bytes, *args), target)\n    eq_(runner._process_remaining_output(None, out, *args), target)",
        "import pytest\nimport tempfile\nimport os\n\n\n@pytest.fixture\ndef path():\n    with tempfile.TemporaryDirectory() as tmpdir:\n        yield tmpdir\n\n\n@pytest.fixture(autouse=True)\ndef setup_git_config():\n    with tempfile.TemporaryDirectory() as home:\n        old_home = os.environ.get(\"HOME\")\n        os.environ[\"HOME\"] = home\n        os.system('git config --global user.name \"DataLad Tester\"')\n        os.system('git config --global user.email \"test@example.com\"')\n        yield\n        if old_home:\n            os.environ[\"HOME\"] = old_home\n"
    ],
    "test_file_names": [
        "test_1.py",
        "conftest.py"
    ],
    "setup_res_code": 0,
    "setup_res_stdout": "Starting Datalad installation attempts...\nVerifying installation...\n❌ Verification failed\nPython 3.9 installation failed, trying Python 3.7...\nVerifying installation...\nDatalad version: 0.9.3.dev935\n✅ Installation successful!\nSuccessfully installed Datalad using Python 3.7\n",
    "setup_res_stderr": "Using CPython 3.9.21\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 25 packages in 91ms\nInstalled 25 packages in 5.60s\n + coverage==7.6.10\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + grabbit==0.2.6\n + importlib-resources==6.4.5\n + iniconfig==2.0.0\n + nibabel==5.3.2\n + num2words==0.5.14\n + numpy==2.0.2\n + packaging==24.2\n + pandas==2.2.3\n + patsy==1.0.1\n + pluggy==1.5.0\n + pybids==0.6.5\n + pytest==8.3.4\n + pytest-cov==6.0.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.13.1\n + setuptools==75.6.0\n + six==1.17.0\n + tomli==2.2.1\n + typing-extensions==4.12.2\n + tzdata==2024.2\n + zipp==3.21.0\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWuXhWK/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWuXhWK/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWuXhWK/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWuXhWK/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpWuXhWK/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nerror: The build backend returned an error\n  Caused by: Call to `setuptools.build_meta:__legacy__.build_editable` failed (exit status: 1)\n\n[stderr]\nTraceback (most recent call last):\n  File \"<string>\", line 14, in <module>\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpzXFGy7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 483, in get_requires_for_build_editable\n    return self.get_requires_for_build_wheel(config_settings)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpzXFGy7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 334, in get_requires_for_build_wheel\n    return self._get_build_requires(config_settings, requirements=[])\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpzXFGy7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 304, in _get_build_requires\n    self.run_setup()\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpzXFGy7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 522, in run_setup\n    super().run_setup(setup_script=setup_script)\n  File \"/home/gcpuser/.cache/uv/builds-v0/.tmpzXFGy7/lib/python3.9/site-packages/setuptools/build_meta.py\", line 320, in run_setup\n    exec(code, locals())\n  File \"<string>\", line 50, in <module>\nAttributeError: module 'platform' has no attribute 'dist'\n\nhint: This usually indicates a problem with the package or the build environment.\nTraceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/datalad/__init__.py\", line 23, in <module>\n    from datalad.cmd import GitRunner\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/datalad/cmd.py\", line 35, in <module>\n    from .utils import on_windows, get_tempfile_kwargs, assure_unicode\n  File \"/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/datalad/utils.py\", line 24, in <module>\n    import wrapt\nModuleNotFoundError: No module named 'wrapt'\nUsing CPython 3.7.9\nCreating virtual environment at: .venv\nActivate with: source .venv/bin/activate\nResolved 26 packages in 406ms\nInstalled 26 packages in 3.69s\n + bids==0.0\n + coverage==7.2.7\n + docopt==0.6.2\n + exceptiongroup==1.2.2\n + fasteners==0.19\n + grabbit==0.2.6\n + importlib-metadata==6.7.0\n + iniconfig==2.0.0\n + nibabel==4.0.2\n + num2words==0.5.14\n + numpy==1.21.6\n + packaging==24.0\n + pandas==1.3.5\n + patsy==1.0.1\n + pluggy==1.2.0\n + pybids==0.6.5\n + pytest==7.4.4\n + pytest-cov==4.1.0\n + python-dateutil==2.9.0.post0\n + pytz==2024.2\n + scipy==1.7.3\n + setuptools==68.0.0\n + six==1.17.0\n + tomli==2.0.1\n + typing-extensions==4.7.1\n + zipp==3.15.0\nResolved 96 packages in 1.48s\nPrepared 1 package in 1.27s\nInstalled 78 packages in 225ms\n + appdirs==1.4.4\n + attrs==24.2.0\n + automat==22.10.0\n + beautifulsoup4==4.12.3\n + boto==2.49.0\n + certifi==2024.12.14\n + cffi==1.15.1\n + chardet==5.2.0\n + charset-normalizer==3.4.1\n + citeproc-py==0.6.0\n + constantly==15.1.0\n + cryptography==44.0.0\n + cssselect==1.2.0\n + datalad==0.10.0.dev1 (from file:///home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e)\n + deprecated==1.2.15\n + duecredit==0.9.3\n + exifread==3.0.0\n + filelock==3.12.2\n + gitdb==4.0.12\n + gitpython==3.1.43\n + httpretty==1.1.4\n + humanize==4.6.0\n + hyperlink==21.0.0\n + idna==3.10\n + importlib-resources==5.12.0\n + incremental==22.10.0\n + iso8601==2.1.0\n + itemadapter==0.8.0\n + itemloaders==1.1.0\n + jaraco-classes==3.2.3\n + jeepney==0.8.0\n + jmespath==1.0.1\n + jsmin==3.0.1\n + keyring==24.1.1\n + keyrings-alt==4.2.0\n + lxml==5.3.0\n + mock==5.1.0\n + more-itertools==9.1.0\n + msgpack==1.0.5\n + multidict==6.0.5\n + mutagen==1.47.0\n + nose==1.3.7\n + parsel==1.8.1\n + patool==1.12\n + pillow==9.5.0\n + protego==0.3.0\n + pyasn1==0.5.1\n + pyasn1-modules==0.3.0\n + pycparser==2.21\n + pydicom==2.4.4\n + pydispatcher==2.0.7\n + pygithub==2.3.0\n + pyjwt==2.8.0\n + pynacl==1.5.0\n + pyopenssl==24.3.0\n + pyperclip==1.9.0\n + python-xmp-toolkit==2.0.2\n + pyyaml==6.0.1\n + queuelib==1.6.2\n + requests==2.31.0\n + requests-file==2.1.0\n + requests-ftp==0.3.1\n + scrapy==2.9.0\n + secretstorage==3.3.3\n + service-identity==21.1.0\n + simplejson==3.19.3\n + smmap==5.0.2\n + soupsieve==2.4.1\n + tldextract==4.0.0\n + tqdm==4.67.1\n + twisted==23.8.0\n + urllib3==2.0.7\n + vcrpy==4.3.0\n + w3lib==2.1.2\n + whoosh==2.7.4\n + wrapt==1.16.0\n + yarl==1.9.4\n + zope-interface==6.4.post2\nResolved 115 packages in 233ms\nInstalled 19 packages in 96ms\n + alabaster==0.7.13\n + babel==2.14.0\n + docutils==0.19\n + imagesize==1.4.1\n + jinja2==3.1.5\n + markupsafe==2.1.5\n + nose-timer==1.0.1\n + pygments==2.17.2\n + pypandoc==1.14\n + snowballstemmer==2.2.0\n + sphinx==5.3.0\n + sphinx-rtd-theme==2.0.0\n + sphinxcontrib-applehelp==1.0.2\n + sphinxcontrib-devhelp==1.0.2\n + sphinxcontrib-htmlhelp==2.0.0\n + sphinxcontrib-jquery==4.1\n + sphinxcontrib-jsmath==1.0.1\n + sphinxcontrib-qthelp==1.0.3\n + sphinxcontrib-serializinghtml==1.1.5\n",
    "new_commit_res_code": 1,
    "new_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e\nplugins: cov-4.1.0\ncollected 12 items\n\nr2e_tests/test_1.py EE....xEEEF.                                         [100%]\n\n==================================== ERRORS ====================================\n______________________ ERROR at setup of test_runner_dry _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 35\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_tempfile\n  def test_runner_dry(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:35\n________________________ ERROR at setup of test_runner _________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 61\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_tempfile\n  def test_runner(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:61\n____________________ ERROR at setup of test_link_file_load _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 229\n  @with_tempfile\n  def test_link_file_load(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:229\n____________________ ERROR at setup of test_runner_failure _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 281\n  @with_tempfile(mkdir=True)\n  def test_runner_failure(dir_):\nE       fixture 'dir_' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:281\n_______________________ ERROR at setup of test_git_path ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 295\n  @with_tempfile(mkdir=True)\n  def test_git_path(dir_):\nE       fixture 'dir_' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:295\n=================================== FAILURES ===================================\n______________________________ test_runner_stdin _______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexnjji5l6h'}\nfilename = '/tmp/datalad_temp_test_runner_stdinsttj_xqk'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_runner_stdin() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n==================================== PASSES ====================================\n____________________________ test_runner_log_stderr ____________________________\n------------------------------ Captured log call -------------------------------\nLevel 9  datalad.cmd:cmd.py:579 Running: echo stderr-Message should be logged >&2\nLevel 9  datalad.cmd:cmd.py:579 stderr| stderr-Message should be logged\nLevel 9  datalad.cmd:cmd.py:579 Running: echo stderr-Message should not be logged >&2\n____________________________ test_runner_log_stdout ____________________________\n------------------------------ Captured log call -------------------------------\nLevel 9  datalad.cmd:cmd.py:579 Running: ['echo', 'stdout-Message should be logged']\nLevel 9  datalad.cmd:cmd.py:579 stdout| stdout-Message should be logged\nLevel 9  datalad.cmd:cmd.py:579 Running: echo stdout-Message should be logged\nLevel 9  datalad.cmd:cmd.py:579 stdout| stdout-Message should be logged\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_runner_instance_callable_dry\nPASSED r2e_tests/test_1.py::test_runner_instance_callable_wet\nPASSED r2e_tests/test_1.py::test_runner_log_stderr\nPASSED r2e_tests/test_1.py::test_runner_log_stdout\nPASSED r2e_tests/test_1.py::test_process_remaining_output\nXFAIL r2e_tests/test_1.py::test_runner_heavy_output - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_runner_heavy_output will be ignored\nERROR r2e_tests/test_1.py::test_runner_dry\nERROR r2e_tests/test_1.py::test_runner\nERROR r2e_tests/test_1.py::test_link_file_load\nERROR r2e_tests/test_1.py::test_runner_failure\nERROR r2e_tests/test_1.py::test_git_path\nFAILED r2e_tests/test_1.py::test_runner_stdin - TypeError: test_runner_stdin(...\n=============== 1 failed, 5 passed, 1 xfailed, 5 errors in 1.23s ===============\n",
    "new_commit_res_stderr": "",
    "old_commit_res_code": 1,
    "old_commit_res_stdout": "============================= test session starts ==============================\nplatform linux -- Python 3.7.9, pytest-7.4.4, pluggy-1.2.0\nrootdir: /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e\nplugins: cov-4.1.0\ncollected 12 items\n\nr2e_tests/test_1.py EE....xEEEFF                                         [100%]\n\n==================================== ERRORS ====================================\n______________________ ERROR at setup of test_runner_dry _______________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 35\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_tempfile\n  def test_runner_dry(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:35\n________________________ ERROR at setup of test_runner _________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 61\n  @ignore_nose_capturing_stdout\n  @assert_cwd_unchanged\n  @with_tempfile\n  def test_runner(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:61\n____________________ ERROR at setup of test_link_file_load _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 229\n  @with_tempfile\n  def test_link_file_load(tempfile):\nE       fixture 'tempfile' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:229\n____________________ ERROR at setup of test_runner_failure _____________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 281\n  @with_tempfile(mkdir=True)\n  def test_runner_failure(dir_):\nE       fixture 'dir_' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:281\n_______________________ ERROR at setup of test_git_path ________________________\nfile /home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py, line 295\n  @with_tempfile(mkdir=True)\n  def test_git_path(dir_):\nE       fixture 'dir_' not found\n>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, path, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, setup_git_config, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory\n>       use 'pytest --fixtures [testpath]' for help on them.\n\n/home/gcpuser/buckets/local_repoeval_bucket/repos/datalad_6f41a9ff01a8ddb5cad728d4bb112d876ecd5b7e/r2e_tests/test_1.py:295\n=================================== FAILURES ===================================\n______________________________ test_runner_stdin _______________________________\n\narg = (), kw = {'path': '/tmp/tmp-page2annexu51lj36n'}\nfilename = '/tmp/datalad_temp_test_runner_stdinipj1n76o'\n\n    @wraps(t)\n    def newfunc(*arg, **kw):\n        with make_tempfile(wrapped=t, **tkwargs) as filename:\n>           return t(*(arg + (filename,)), **kw)\nE           TypeError: test_runner_stdin() got multiple values for argument 'path'\n\ndatalad/tests/utils.py:664: TypeError\n________________________ test_process_remaining_output _________________________\n\n    def test_process_remaining_output():\n        runner = Runner()\n        out = u\"\"\"\\\n    s\n    п\n    \"\"\"\n        out_bytes = out.encode('utf-8')\n        target = u\"s\\nп\\n\".encode('utf-8')\n        args = ['stdout', None, False, False]\n        #  probably #2185\n>       eq_(runner._process_remaining_output(None, out_bytes, *args), target)\n\nr2e_tests/test_1.py:324: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\na = b's\\xd0\\xbf', b = b's\\n\\xd0\\xbf\\n', msg = None\n\n    def eq_(a, b, msg=None):\n        \"\"\"Shorthand for 'assert a == b, \"%r != %r\" % (a, b)\n        \"\"\"\n        if not a == b:\n>           raise AssertionError(msg or \"%r != %r\" % (a, b))\nE           AssertionError: b's\\xd0\\xbf' != b's\\n\\xd0\\xbf\\n'\n\n.venv/lib/python3.7/site-packages/nose/tools/trivial.py:29: AssertionError\n==================================== PASSES ====================================\n____________________________ test_runner_log_stderr ____________________________\n------------------------------ Captured log call -------------------------------\nLevel 9  datalad.cmd:cmd.py:578 Running: echo stderr-Message should be logged >&2\nLevel 9  datalad.cmd:cmd.py:578 stderr| stderr-Message should be logged\nLevel 9  datalad.cmd:cmd.py:578 Running: echo stderr-Message should not be logged >&2\n____________________________ test_runner_log_stdout ____________________________\n------------------------------ Captured log call -------------------------------\nLevel 9  datalad.cmd:cmd.py:578 Running: ['echo', 'stdout-Message should be logged']\nLevel 9  datalad.cmd:cmd.py:578 stdout| stdout-Message should be logged\nLevel 9  datalad.cmd:cmd.py:578 Running: echo stdout-Message should be logged\nLevel 9  datalad.cmd:cmd.py:578 stdout| stdout-Message should be logged\n=========================== short test summary info ============================\nPASSED r2e_tests/test_1.py::test_runner_instance_callable_dry\nPASSED r2e_tests/test_1.py::test_runner_instance_callable_wet\nPASSED r2e_tests/test_1.py::test_runner_log_stderr\nPASSED r2e_tests/test_1.py::test_runner_log_stdout\nXFAIL r2e_tests/test_1.py::test_runner_heavy_output - reason: [NOTRUN] yield tests were removed in pytest 4.0 - test_runner_heavy_output will be ignored\nERROR r2e_tests/test_1.py::test_runner_dry\nERROR r2e_tests/test_1.py::test_runner\nERROR r2e_tests/test_1.py::test_link_file_load\nERROR r2e_tests/test_1.py::test_runner_failure\nERROR r2e_tests/test_1.py::test_git_path\nFAILED r2e_tests/test_1.py::test_runner_stdin - TypeError: test_runner_stdin(...\nFAILED r2e_tests/test_1.py::test_process_remaining_output - AssertionError: b...\n=============== 2 failed, 4 passed, 1 xfailed, 5 errors in 0.73s ===============\n",
    "old_commit_res_stderr": ""
}
datalad/cmd.py datalad/tests/test_cmd.py
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Missing newline character at the end of processed output lines causes assertion failure

**Description:**
When processing output lines, the lines returned do not consistently end with a newline character (`\n`). This leads to unexpected behavior in downstream processes that rely on each line being properly terminated.

**Example:**
```python
def process_output():
    out = process_remaining_output(b's\xd0\xbf')
    print(out)

def process_remaining_output(data):
    # Expected to ensure each line ends with a newline
    return data  # Missing newline characters

process_output()
```
In the example above, the processed output `b's\xd0\xbf'` lacks the trailing newline characters, whereas the expected output should be `b's\n\xd0\xbf\n'`.

**Expected Behavior:**
Each line returned after processing should end with a newline character (`\n`). For example, given the input `b's\xd0\xbf'`, the output should be `b's\n\xd0\xbf\n'`.

**Actual Behavior:**
The processed output does not include the required newline characters, resulting in `b's\xd0\xbf'` instead of the expected `b's\n\xd0\xbf\n'`. This discrepancy causes assertion failures in tests that validate the presence of newline characters at the end of each line.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/cmd.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""
Wrapper for command and function calls, allowing for dry runs and output handling

"""


import subprocess
import sys
import logging
import os
import shutil
import shlex
import atexit
import functools
import tempfile

from collections import OrderedDict
from six import PY3, PY2
from six import string_types, binary_type, text_type
from os.path import abspath, isabs, pathsep, exists

from .consts import GIT_SSH_COMMAND
from .dochelpers import exc_str
from .support.exceptions import CommandError
from .support.protocol import NullProtocol, DryRunProtocol, \
    ExecutionTimeProtocol, ExecutionTimeExternalsProtocol
from .utils import on_windows, get_tempfile_kwargs, assure_unicode
from .dochelpers import borrowdoc

lgr = logging.getLogger('datalad.cmd')

# In python3 to split byte stream on newline, it must be bytes
linesep_bytes = os.linesep.encode()

_TEMP_std = sys.stdout, sys.stderr
# To be used in the temp file name to distinguish the ones we create
# in Runner so we take care about their removal, in contrast to those
# which might be created outside and passed into Runner
_MAGICAL_OUTPUT_MARKER = "_runneroutput_"

if PY2:
    # TODO apparently there is a recommended substitution for Python2
    # which is a backported implementation of python3 subprocess
    # https://pypi.python.org/pypi/subprocess32/
    file_class = file
else:
    from io import IOBase as file_class


def _decide_to_log(v):
    """Hacky workaround for now so we could specify per each which to
    log online and which to the log"""
    if isinstance(v, bool) or callable(v):
        return v
    elif v in {'online'}:
        return True
    elif v in {'offline'}:
        return False
    else:
        raise ValueError("can be bool, callable, 'online' or 'offline'")


def _get_output_stream(log_std, false_value):
    """Helper to prepare output stream for Popen and use file for 'offline'

    Necessary to avoid lockdowns when both stdout and stderr are pipes
    """
    if log_std:
        if log_std == 'offline':
            # we will open a temporary file

            tf = tempfile.mktemp(
                **get_tempfile_kwargs({}, prefix=_MAGICAL_OUTPUT_MARKER)
            )
            return open(tf, 'w')  # XXX PY3 should be 'b' may be?
        else:
            return subprocess.PIPE
    else:
        return false_value


def _cleanup_output(stream, std):
    if isinstance(stream, file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:
        if not stream.closed:
            stream.close()
        if exists(stream.name):
            os.unlink(stream.name)
    elif stream == subprocess.PIPE:
        std.close()


class Runner(object):
    """Provides a wrapper for calling functions and commands.

    An object of this class provides a methods that calls shell commands or
    python functions, allowing for protocolling the calls and output handling.

    Outputs (stdout and stderr) can be either logged or streamed to system's
    stdout/stderr during execution.
    This can be enabled or disabled for both of them independently.
    Additionally, a protocol object can be a used with the Runner. Such a
    protocol has to implement datalad.support.protocol.ProtocolInterface, is
    able to record calls and allows for dry runs.
    """

    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',
                 '_log_opts']

    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):
        """
        Parameters
        ----------
        cwd: string, optional
             Base current working directory for commands.  Could be overridden
             per run call via cwd option
        env: dict, optional
             Custom environment to use for calls. Could be overridden per run
             call via env option
        protocol: ProtocolInterface
             Protocol object to write to.
        log_outputs : bool, optional
             Switch to instruct either outputs should be logged or not.  If not
             set (default), config 'datalad.log.outputs' would be consulted
        """

        self.cwd = cwd
        self.env = env
        if protocol is None:
            # TODO: config cmd.protocol = null
            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')
            protocol = {
                'externals-time': ExecutionTimeExternalsProtocol,
                'time': ExecutionTimeProtocol,
                'null': NullProtocol
            }[protocol_str]()
            if protocol_str != 'null':
                # we need to dump it into a file at the end
                # TODO: config cmd.protocol_prefix = protocol
                filename = '%s-%s.log' % (
                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),
                    id(self)
                )
                atexit.register(functools.partial(protocol.write_to_file, filename))

        self.protocol = protocol
        # Various options for logging
        self._log_opts = {}
        # we don't know yet either we need ot log every output or not
        if log_outputs is not None:
            self._log_opts['outputs'] = log_outputs

    def __call__(self, cmd, *args, **kwargs):
        """Convenience method

        This will call run() or call() depending on the kind of `cmd`.
        If `cmd` is a string it is interpreted as the to be executed command.
        Otherwise it is expected to be a callable.
        Any other argument is passed to the respective method.

        Parameters
        ----------
        cmd: str or callable
           command string to be executed via shell or callable to be called.

        `*args`:
        `**kwargs`:
           see Runner.run() and Runner.call() for available arguments.

        Raises
        ------
        TypeError
          if cmd is neither a string nor a callable.
        """

        if isinstance(cmd, string_types) or isinstance(cmd, list):
            return self.run(cmd, *args, **kwargs)
        elif callable(cmd):
            return self.call(cmd, *args, **kwargs)
        else:
            raise TypeError("Argument 'command' is neither a string, "
                            "nor a list nor a callable.")

    def _opt_env_adapter(v):
        """If value is a string, split by ,"""
        if v:
            if v.isdigit():
                log_env = bool(int(v))
            else:
                log_env = v.split(',')
            return log_env
        else:
            return False

    _LOG_OPTS_ADAPTERS = OrderedDict([
        ('outputs', None),
        ('cwd', None),
        ('env', _opt_env_adapter),
        ('stdin', None),
    ])

    def _get_log_setting(self, opt, default=False):
        try:
            return self._log_opts[opt]
        except KeyError:
            try:
                from . import cfg
            except ImportError:
                return default
            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)
            self._log_opts[opt] = \
                (cfg.getbool if not adapter else cfg.get_value)(
                    'datalad.log.cmd', opt, default=default)
            if adapter:
                self._log_opts[opt] = adapter(self._log_opts[opt])
            return self._log_opts[opt]

    @property
    def log_outputs(self):
        return self._get_log_setting('outputs')

    @property
    def log_cwd(self):
        return self._get_log_setting('cwd')

    @property
    def log_stdin(self):
        return self._get_log_setting('stdin')

    @property
    def log_env(self):
        return self._get_log_setting('env')

    # Two helpers to encapsulate formatting/output
    def _log_out(self, line):
        if line and self.log_outputs:
            self.log("stdout| " + line.rstrip('\n'))

    def _log_err(self, line, expected=False):
        if line and self.log_outputs:
            self.log("stderr| " + line.rstrip('\n'),
                     level={True: 9,
                            False: 11}[expected])

    def _get_output_online(self, proc,
                           log_stdout, log_stderr,
                           outputstream, errstream,
                           expect_stderr=False, expect_fail=False):
        """

        If log_stdout or log_stderr are callables, they will be given a read
        line to be processed, and return processed result.  So if they need to
        'swallow' the line from being logged, should just return None

        Parameters
        ----------
        proc
        log_stdout: bool or callable or 'online' or 'offline'
        log_stderr: : bool or callable or 'online' or 'offline'
          If any of those 'offline', we would call proc.communicate at the
          end to grab possibly outstanding output from it
        expect_stderr
        expect_fail

        Returns
        -------

        """
        stdout, stderr = binary_type(), binary_type()

        log_stdout_ = _decide_to_log(log_stdout)
        log_stderr_ = _decide_to_log(log_stderr)
        log_stdout_is_callable = callable(log_stdout_)
        log_stderr_is_callable = callable(log_stderr_)

        # arguments to be passed into _process_one_line
        stdout_args = (
                'stdout',
                proc, log_stdout_, log_stdout_is_callable
        )
        stderr_args = (
                'stderr',
                proc, log_stderr_, log_stderr_is_callable,
                expect_stderr or expect_fail
        )

        while proc.poll() is None:
            # see for a possibly useful approach to processing output
            # in another thread http://codereview.stackexchange.com/a/17959
            # current problem is that if there is no output on stderr
            # it stalls
            if log_stdout_:
                stdout += self._process_one_line(*stdout_args)
            if log_stderr_:
                stderr += self._process_one_line(*stderr_args)

        # Handle possible remaining output
        stdout_, stderr_ = proc.communicate()
        # ??? should we condition it on log_stdout in {'offline'} ???
        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)
        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)

        return stdout, stderr

    def _process_remaining_output(self, stream, out_, *pargs):
        """Helper to process output which might have been obtained from popen or
        should be loaded from file"""
        out = binary_type()
        if isinstance(stream,
                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:
            assert out_ is None, "should have gone into a file"
            if not stream.closed:
                stream.close()
            with open(stream.name, 'rb') as f:
                for line in f:
                    out += self._process_one_line(*pargs, line=line)
        else:
            if out_:
                # resolving a once in a while failing test #2185
                if isinstance(out_, text_type):
                    out_ = out_.encode('utf-8')
                for line in out_.split(linesep_bytes):
                    out += self._process_one_line(*pargs, line=line)
        return out

    def _process_one_line(self, out_type, proc, log_, log_is_callable,
                          expected=False, line=None):
        if line is None:
            lgr.log(3, "Reading line from %s", out_type)
            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()
        else:
            lgr.log(3, "Processing provided line")
        if line and log_is_callable:
            # Let it be processed
            line = log_(assure_unicode(line))
            if line is not None:
                # we are working with binary type here
                line = line.encode()
        if line:
            if out_type == 'stdout':
                self._log_out(assure_unicode(line))
            elif out_type == 'stderr':
                self._log_err(line.decode() if PY3 else line,
                              expected)
            else:  # pragma: no cover
                raise RuntimeError("must not get here")
            return line
        # it was output already directly but for code to work, return ""
        return binary_type()

    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,
            expect_stderr=False, expect_fail=False,
            cwd=None, env=None, shell=None, stdin=None):
        """Runs the command `cmd` using shell.

        In case of dry-mode `cmd` is just added to `commands` and it is
        actually executed otherwise.
        Allows for separately logging stdout and stderr  or streaming it to
        system's stdout or stderr respectively.

        Note: Using a string as `cmd` and shell=True allows for piping,
              multiple commands, etc., but that implies shlex.split() is not
              used. This is considered to be a security hazard.
              So be careful with input.

        Parameters
        ----------
        cmd : str, list
          String (or list) defining the command call.  No shell is used if cmd
          is specified as a list

        log_stdout: bool, optional
            If True, stdout is logged. Goes to sys.stdout otherwise.

        log_stderr: bool, optional
            If True, stderr is logged. Goes to sys.stderr otherwise.

        log_online: bool, optional
            Either to log as output comes in.  Setting to True is preferable
            for running user-invoked actions to provide timely output

        expect_stderr: bool, optional
            Normally, having stderr output is a signal of a problem and thus it
            gets logged at level 11.  But some utilities, e.g. wget, use
            stderr for their progress output.  Whenever such output is expected,
            set it to True and output will be logged at level 9 unless
            exit status is non-0 (in non-online mode only, in online -- would
            log at 9)

        expect_fail: bool, optional
            Normally, if command exits with non-0 status, it is considered an
            error and logged at level 11 (above DEBUG). But if the call intended
            for checking routine, such messages are usually not needed, thus
            it will be logged at level 9.

        cwd : string, optional
            Directory under which run the command (passed to Popen)

        env : string, optional
            Custom environment to pass

        shell: bool, optional
            Run command in a shell.  If not specified, then it runs in a shell
            only if command is specified as a string (not a list)

        stdin: file descriptor
            input stream to connect to stdin of the process.

        Returns
        -------
        (stdout, stderr)

        Raises
        ------
        CommandError
           if command's exitcode wasn't 0 or None. exitcode is passed to
           CommandError's `code`-field. Command's stdout and stderr are stored
           in CommandError's `stdout` and `stderr` fields respectively.
        """
        outputstream = _get_output_stream(log_stdout, sys.stdout)
        errstream = _get_output_stream(log_stderr, sys.stderr)

        popen_env = env or self.env

        # TODO: if outputstream is sys.stdout and that one is set to StringIO
        #       we have to "shim" it with something providing fileno().
        # This happens when we do not swallow outputs, while allowing nosetest's
        # StringIO to be provided as stdout, crashing the Popen requiring
        # fileno().  In out swallow_outputs, we just use temporary files
        # to overcome this problem.
        # For now necessary test code should be wrapped into swallow_outputs cm
        # to avoid the problem
        log_msgs = ["Running: %s"]
        log_args = [cmd]
        if self.log_cwd:
            log_msgs += ['cwd=%r']
            log_args += [cwd or self.cwd]
        if self.log_stdin:
            log_msgs += ['stdin=%r']
            log_args += [stdin]
        log_env = self.log_env
        if log_env and popen_env:
            log_msgs += ["env=%r"]
            log_args.append(
                popen_env if log_env is True
                else {k: popen_env[k] for k in log_env if k in popen_env}
            )
        log_msg = '\n'.join(log_msgs)
        self.log(log_msg, *log_args)

        if self.protocol.do_execute_ext_commands:

            if shell is None:
                shell = isinstance(cmd, string_types)

            if self.protocol.records_ext_commands:
                prot_exc = None
                prot_id = self.protocol.start_section(
                    shlex.split(cmd, posix=not on_windows)
                    if isinstance(cmd, string_types)
                    else cmd)
            try:
                proc = subprocess.Popen(cmd,
                                        stdout=outputstream,
                                        stderr=errstream,
                                        shell=shell,
                                        cwd=cwd or self.cwd,
                                        env=popen_env,
                                        stdin=stdin)

            except Exception as e:
                prot_exc = e
                lgr.log(11, "Failed to start %r%r: %s" %
                        (cmd, " under %r" % cwd if cwd else '', exc_str(e)))
                raise

            finally:
                if self.protocol.records_ext_commands:
                    self.protocol.end_section(prot_id, prot_exc)

            try:
                if log_online:
                    out = self._get_output_online(proc,
                                                  log_stdout, log_stderr,
                                                  outputstream, errstream,
                                                  expect_stderr=expect_stderr,
                                                  expect_fail=expect_fail)
                else:
                    out = proc.communicate()

                if PY3:
                    # Decoding was delayed to this point
                    def decode_if_not_None(x):
                        return "" if x is None else binary_type.decode(x)
                    # TODO: check if we can avoid PY3 specific here
                    out = tuple(map(decode_if_not_None, out))

                status = proc.poll()

                # needs to be done after we know status
                if not log_online:
                    self._log_out(out[0])
                    if status not in [0, None]:
                        self._log_err(out[1], expected=expect_fail)
                    else:
                        # as directed
                        self._log_err(out[1], expected=expect_stderr)

                if status not in [0, None]:
                    msg = "Failed to run %r%s. Exit code=%d. out=%s err=%s" \
                        % (cmd, " under %r" % (cwd or self.cwd), status, out[0], out[1])
                    lgr.log(9 if expect_fail else 11, msg)
                    raise CommandError(str(cmd), msg, status, out[0], out[1])
                else:
                    self.log("Finished running %r with status %s" % (cmd, status),
                             level=8)
            finally:
                # Those streams are for us to close if we asked for a PIPE
                # TODO -- assure closing the files import pdb; pdb.set_trace()
                _cleanup_output(outputstream, proc.stdout)
                _cleanup_output(errstream, proc.stderr)

        else:
            if self.protocol.records_ext_commands:
                self.protocol.add_section(shlex.split(cmd,
                                                      posix=not on_windows)
                                          if isinstance(cmd, string_types)
                                          else cmd, None)
            out = ("DRY", "DRY")

        return out

    def call(self, f, *args, **kwargs):
        """Helper to unify collection of logging all "dry" actions.

        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with
        its arguments to `commands` otherwise.

        Parameters
        ----------
        f: callable
        """
        if self.protocol.do_execute_callables:
            if self.protocol.records_callables:
                prot_exc = None
                prot_id = self.protocol.start_section(
                    [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)])

            try:
                return f(*args, **kwargs)
            except Exception as e:
                prot_exc = e
                raise
            finally:
                if self.protocol.records_callables:
                    self.protocol.end_section(prot_id, prot_exc)
        else:
            if self.protocol.records_callables:
                self.protocol.add_section(
                    [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)],
                    None)

    def log(self, msg, *args, **kwargs):
        """log helper

        Logs at level 9 by default and adds "Protocol:"-prefix in order to
        log the used protocol.
        """
        level = kwargs.pop('level', 9)
        if isinstance(self.protocol, NullProtocol):
            lgr.log(level, msg, *args, **kwargs)
        else:
            if args:
                msg = msg % args
            lgr.log(level, "{%s} %s" % (
                self.protocol.__class__.__name__, msg)
            )


class GitRunner(Runner):
    """
    Runner to be used to run git and git annex commands

    Overloads the runner class to check & update GIT_DIR and
    GIT_WORK_TREE environment variables set to the absolute path
    if is defined and is relative path
    """
    _GIT_PATH = None

    @borrowdoc(Runner)
    def __init__(self, *args, **kwargs):
        super(GitRunner, self).__init__(*args, **kwargs)
        self._check_git_path()

    @staticmethod
    def _check_git_path():
        """If using bundled git-annex, we would like to use bundled with it git

        Thus we will store _GIT_PATH a path to git in the same directory as annex
        if found.  If it is empty (but not None), we do nothing
        """
        if GitRunner._GIT_PATH is None:
            from distutils.spawn import find_executable
            # with all the nesting of config and this runner, cannot use our
            # cfg here, so will resort to dark magic of environment options
            if (os.environ.get('DATALAD_USE_DEFAULT_GIT', '0').lower()
                    in ('1', 'on', 'true', 'yes')):
                git_fpath = find_executable("git")
                if git_fpath:
                    GitRunner._GIT_PATH = ''
                    lgr.log(9, "Will use default git %s", git_fpath)
                    return  # we are done - there is a default git avail.
                # if not -- we will look for a bundled one

            annex_fpath = find_executable("git-annex")
            if not annex_fpath:
                # not sure how to live further anyways! ;)
                alongside = False
            else:
                annex_path = os.path.dirname(os.path.realpath(annex_fpath))
                if on_windows:
                    # just bundled installations so git should be taken from annex
                    alongside = True
                else:
                    alongside = os.path.lexists(os.path.join(annex_path, 'git'))
            GitRunner._GIT_PATH = annex_path if alongside else ''
            lgr.log(9, "Will use git under %r (no adjustments to PATH if empty "
                       "string)", GitRunner._GIT_PATH)
            assert(GitRunner._GIT_PATH is not None)  # we made the decision!

    @staticmethod
    def get_git_environ_adjusted(env=None):
        """
        Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined
        """
        # if env set copy else get os environment
        git_env = env.copy() if env else os.environ.copy()
        if GitRunner._GIT_PATH:
            git_env['PATH'] = pathsep.join([GitRunner._GIT_PATH, git_env['PATH']]) \
                if 'PATH' in git_env \
                else GitRunner._GIT_PATH

        for varstring in ['GIT_DIR', 'GIT_WORK_TREE']:
            var = git_env.get(varstring)
            if var:                                    # if env variable set
                if not isabs(var):                     # and it's a relative path
                    git_env[varstring] = abspath(var)  # to absolute path
                    lgr.log(9, "Updated %s to %s", varstring, git_env[varstring])

        if 'GIT_SSH_COMMAND' not in git_env:
            git_env['GIT_SSH_COMMAND'] = GIT_SSH_COMMAND

        return git_env

    def run(self, cmd, env=None, *args, **kwargs):
        return super(GitRunner, self).run(
            cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)


# ####
# Preserve from previous version
# TODO: document intention
# ####
# this one might get under Runner for better output/control
def link_file_load(src, dst, dry_run=False):
    """Just a little helper to hardlink files's load
    """
    dst_dir = os.path.dirname(dst)
    if not os.path.exists(dst_dir):
        os.makedirs(dst_dir)
    if os.path.lexists(dst):
        lgr.log(9, "Destination file %(dst)s exists. Removing it first", locals())
        # TODO: how would it interact with git/git-annex
        os.unlink(dst)
    lgr.log(9, "Hardlinking %(src)s under %(dst)s", locals())
    src_realpath = os.path.realpath(src)

    try:
        os.link(src_realpath, dst)
    except AttributeError as e:
        lgr.warn("Linking of %s failed (%s), copying file" % (src, e))
        shutil.copyfile(src_realpath, dst)
        shutil.copystat(src_realpath, dst)
    else:
        lgr.log(2, "Hardlinking finished")


def get_runner(*args, **kwargs):
    # needs local import, because the ConfigManager itself needs the runner
    from . import cfg
    # TODO:  this is all crawl specific -- should be moved away
    if cfg.obtain('datalad.crawl.dryrun', default=False):
        kwargs = kwargs.copy()
        kwargs['protocol'] = DryRunProtocol()
    return Runner(*args, **kwargs)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
[ISSUE]
**Title:** Missing newline character at the end of processed output lines causes assertion failure

**Description:**
When processing output lines, the lines returned do not consistently end with a newline character (`\n`). This leads to unexpected behavior in downstream processes that rely on each line being properly terminated.

**Example:**
```python
def process_output():
    out = process_remaining_output(b's\xd0\xbf')
    print(out)

def process_remaining_output(data):
    # Expected to ensure each line ends with a newline
    return data  # Missing newline characters

process_output()
```
In the example above, the processed output `b's\xd0\xbf'` lacks the trailing newline characters, whereas the expected output should be `b's\n\xd0\xbf\n'`.

**Expected Behavior:**
Each line returned after processing should end with a newline character (`\n`). For example, given the input `b's\xd0\xbf'`, the output should be `b's\n\xd0\xbf\n'`.

**Actual Behavior:**
The processed output does not include the required newline characters, resulting in `b's\xd0\xbf'` instead of the expected `b's\n\xd0\xbf\n'`. This discrepancy causes assertion failures in tests that validate the presence of newline characters at the end of each line.

[/ISSUE]
{
    "test_runner_instance_callable_dry": "PASSED",
    "test_runner_instance_callable_wet": "PASSED",
    "test_runner_log_stderr": "PASSED",
    "test_runner_log_stdout": "PASSED",
    "test_process_remaining_output": "PASSED",
    "test_runner_dry": "ERROR",
    "test_runner": "ERROR",
    "test_link_file_load": "ERROR",
    "test_runner_failure": "ERROR",
    "test_git_path": "ERROR",
    "test_runner_stdin": "FAILED"
}
diff --git a/datalad/cmd.py b/datalad/cmd.py
index be04eb5cd..8d5fb026e 100644
--- a/datalad/cmd.py
+++ b/datalad/cmd.py
@@ -326,8 +326,9 @@ class Runner(object):
                 # resolving a once in a while failing test #2185
                 if isinstance(out_, text_type):
                     out_ = out_.encode('utf-8')
-                for line in out_.split(linesep_bytes):
-                    out += self._process_one_line(*pargs, line=line)
+                out += linesep_bytes.join(
+                    self._process_one_line(*pargs, line=line)
+                    for line in out_.split(linesep_bytes))
         return out
 
     def _process_one_line(self, out_type, proc, log_, log_is_callable,

{"datalad/cmd.py": "# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-\n# ex: set sts=4 ts=4 sw=4 noet:\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n#\n#   See COPYING file distributed along with the datalad package for the\n#   copyright and license terms.\n#\n# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##\n\"\"\"\nWrapper for command and function calls, allowing for dry runs and output handling\n\n\"\"\"\n\n\nimport subprocess\nimport sys\nimport logging\nimport os\nimport shutil\nimport shlex\nimport atexit\nimport functools\nimport tempfile\n\nfrom collections import OrderedDict\nfrom six import PY3, PY2\nfrom six import string_types, binary_type, text_type\nfrom os.path import abspath, isabs, pathsep, exists\n\nfrom .consts import GIT_SSH_COMMAND\nfrom .dochelpers import exc_str\nfrom .support.exceptions import CommandError\nfrom .support.protocol import NullProtocol, DryRunProtocol, \\\n    ExecutionTimeProtocol, ExecutionTimeExternalsProtocol\nfrom .utils import on_windows, get_tempfile_kwargs, assure_unicode\nfrom .dochelpers import borrowdoc\n\nlgr = logging.getLogger('datalad.cmd')\n\n# In python3 to split byte stream on newline, it must be bytes\nlinesep_bytes = os.linesep.encode()\n\n_TEMP_std = sys.stdout, sys.stderr\n# To be used in the temp file name to distinguish the ones we create\n# in Runner so we take care about their removal, in contrast to those\n# which might be created outside and passed into Runner\n_MAGICAL_OUTPUT_MARKER = \"_runneroutput_\"\n\nif PY2:\n    # TODO apparently there is a recommended substitution for Python2\n    # which is a backported implementation of python3 subprocess\n    # https://pypi.python.org/pypi/subprocess32/\n    file_class = file\nelse:\n    from io import IOBase as file_class\n\n\ndef _decide_to_log(v):\n    \"\"\"Hacky workaround for now so we could specify per each which to\n    log online and which to the log\"\"\"\n    if isinstance(v, bool) or callable(v):\n        return v\n    elif v in {'online'}:\n        return True\n    elif v in {'offline'}:\n        return False\n    else:\n        raise ValueError(\"can be bool, callable, 'online' or 'offline'\")\n\n\ndef _get_output_stream(log_std, false_value):\n    \"\"\"Helper to prepare output stream for Popen and use file for 'offline'\n\n    Necessary to avoid lockdowns when both stdout and stderr are pipes\n    \"\"\"\n    if log_std:\n        if log_std == 'offline':\n            # we will open a temporary file\n\n            tf = tempfile.mktemp(\n                **get_tempfile_kwargs({}, prefix=_MAGICAL_OUTPUT_MARKER)\n            )\n            return open(tf, 'w')  # XXX PY3 should be 'b' may be?\n        else:\n            return subprocess.PIPE\n    else:\n        return false_value\n\n\ndef _cleanup_output(stream, std):\n    if isinstance(stream, file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n        if not stream.closed:\n            stream.close()\n        if exists(stream.name):\n            os.unlink(stream.name)\n    elif stream == subprocess.PIPE:\n        std.close()\n\n\nclass Runner(object):\n    \"\"\"Provides a wrapper for calling functions and commands.\n\n    An object of this class provides a methods that calls shell commands or\n    python functions, allowing for protocolling the calls and output handling.\n\n    Outputs (stdout and stderr) can be either logged or streamed to system's\n    stdout/stderr during execution.\n    This can be enabled or disabled for both of them independently.\n    Additionally, a protocol object can be a used with the Runner. Such a\n    protocol has to implement datalad.support.protocol.ProtocolInterface, is\n    able to record calls and allows for dry runs.\n    \"\"\"\n\n    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',\n                 '_log_opts']\n\n    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):\n        \"\"\"\n        Parameters\n        ----------\n        cwd: string, optional\n             Base current working directory for commands.  Could be overridden\n             per run call via cwd option\n        env: dict, optional\n             Custom environment to use for calls. Could be overridden per run\n             call via env option\n        protocol: ProtocolInterface\n             Protocol object to write to.\n        log_outputs : bool, optional\n             Switch to instruct either outputs should be logged or not.  If not\n             set (default), config 'datalad.log.outputs' would be consulted\n        \"\"\"\n\n        self.cwd = cwd\n        self.env = env\n        if protocol is None:\n            # TODO: config cmd.protocol = null\n            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')\n            protocol = {\n                'externals-time': ExecutionTimeExternalsProtocol,\n                'time': ExecutionTimeProtocol,\n                'null': NullProtocol\n            }[protocol_str]()\n            if protocol_str != 'null':\n                # we need to dump it into a file at the end\n                # TODO: config cmd.protocol_prefix = protocol\n                filename = '%s-%s.log' % (\n                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),\n                    id(self)\n                )\n                atexit.register(functools.partial(protocol.write_to_file, filename))\n\n        self.protocol = protocol\n        # Various options for logging\n        self._log_opts = {}\n        # we don't know yet either we need ot log every output or not\n        if log_outputs is not None:\n            self._log_opts['outputs'] = log_outputs\n\n    def __call__(self, cmd, *args, **kwargs):\n        \"\"\"Convenience method\n\n        This will call run() or call() depending on the kind of `cmd`.\n        If `cmd` is a string it is interpreted as the to be executed command.\n        Otherwise it is expected to be a callable.\n        Any other argument is passed to the respective method.\n\n        Parameters\n        ----------\n        cmd: str or callable\n           command string to be executed via shell or callable to be called.\n\n        `*args`:\n        `**kwargs`:\n           see Runner.run() and Runner.call() for available arguments.\n\n        Raises\n        ------\n        TypeError\n          if cmd is neither a string nor a callable.\n        \"\"\"\n\n        if isinstance(cmd, string_types) or isinstance(cmd, list):\n            return self.run(cmd, *args, **kwargs)\n        elif callable(cmd):\n            return self.call(cmd, *args, **kwargs)\n        else:\n            raise TypeError(\"Argument 'command' is neither a string, \"\n                            \"nor a list nor a callable.\")\n\n    def _opt_env_adapter(v):\n        \"\"\"If value is a string, split by ,\"\"\"\n        if v:\n            if v.isdigit():\n                log_env = bool(int(v))\n            else:\n                log_env = v.split(',')\n            return log_env\n        else:\n            return False\n\n    _LOG_OPTS_ADAPTERS = OrderedDict([\n        ('outputs', None),\n        ('cwd', None),\n        ('env', _opt_env_adapter),\n        ('stdin', None),\n    ])\n\n    def _get_log_setting(self, opt, default=False):\n        try:\n            return self._log_opts[opt]\n        except KeyError:\n            try:\n                from . import cfg\n            except ImportError:\n                return default\n            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)\n            self._log_opts[opt] = \\\n                (cfg.getbool if not adapter else cfg.get_value)(\n                    'datalad.log.cmd', opt, default=default)\n            if adapter:\n                self._log_opts[opt] = adapter(self._log_opts[opt])\n            return self._log_opts[opt]\n\n    @property\n    def log_outputs(self):\n        return self._get_log_setting('outputs')\n\n    @property\n    def log_cwd(self):\n        return self._get_log_setting('cwd')\n\n    @property\n    def log_stdin(self):\n        return self._get_log_setting('stdin')\n\n    @property\n    def log_env(self):\n        return self._get_log_setting('env')\n\n    # Two helpers to encapsulate formatting/output\n    def _log_out(self, line):\n        if line and self.log_outputs:\n            self.log(\"stdout| \" + line.rstrip('\\n'))\n\n    def _log_err(self, line, expected=False):\n        if line and self.log_outputs:\n            self.log(\"stderr| \" + line.rstrip('\\n'),\n                     level={True: 9,\n                            False: 11}[expected])\n\n    def _get_output_online(self, proc,\n                           log_stdout, log_stderr,\n                           outputstream, errstream,\n                           expect_stderr=False, expect_fail=False):\n        \"\"\"\n\n        If log_stdout or log_stderr are callables, they will be given a read\n        line to be processed, and return processed result.  So if they need to\n        'swallow' the line from being logged, should just return None\n\n        Parameters\n        ----------\n        proc\n        log_stdout: bool or callable or 'online' or 'offline'\n        log_stderr: : bool or callable or 'online' or 'offline'\n          If any of those 'offline', we would call proc.communicate at the\n          end to grab possibly outstanding output from it\n        expect_stderr\n        expect_fail\n\n        Returns\n        -------\n\n        \"\"\"\n        stdout, stderr = binary_type(), binary_type()\n\n        log_stdout_ = _decide_to_log(log_stdout)\n        log_stderr_ = _decide_to_log(log_stderr)\n        log_stdout_is_callable = callable(log_stdout_)\n        log_stderr_is_callable = callable(log_stderr_)\n\n        # arguments to be passed into _process_one_line\n        stdout_args = (\n                'stdout',\n                proc, log_stdout_, log_stdout_is_callable\n        )\n        stderr_args = (\n                'stderr',\n                proc, log_stderr_, log_stderr_is_callable,\n                expect_stderr or expect_fail\n        )\n\n        while proc.poll() is None:\n            # see for a possibly useful approach to processing output\n            # in another thread http://codereview.stackexchange.com/a/17959\n            # current problem is that if there is no output on stderr\n            # it stalls\n            if log_stdout_:\n                stdout += self._process_one_line(*stdout_args)\n            if log_stderr_:\n                stderr += self._process_one_line(*stderr_args)\n\n        # Handle possible remaining output\n        stdout_, stderr_ = proc.communicate()\n        # ??? should we condition it on log_stdout in {'offline'} ???\n        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)\n        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)\n\n        return stdout, stderr\n\n    def _process_remaining_output(self, stream, out_, *pargs):\n        \"\"\"Helper to process output which might have been obtained from popen or\n        should be loaded from file\"\"\"\n        out = binary_type()\n        if isinstance(stream,\n                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:\n            assert out_ is None, \"should have gone into a file\"\n            if not stream.closed:\n                stream.close()\n            with open(stream.name, 'rb') as f:\n                for line in f:\n                    out += self._process_one_line(*pargs, line=line)\n        else:\n            if out_:\n                # resolving a once in a while failing test #2185\n                if isinstance(out_, text_type):\n                    out_ = out_.encode('utf-8')\n                for line in out_.split(linesep_bytes):\n                    out += self._process_one_line(*pargs, line=line)\n        return out\n\n    def _process_one_line(self, out_type, proc, log_, log_is_callable,\n                          expected=False, line=None):\n        if line is None:\n            lgr.log(3, \"Reading line from %s\", out_type)\n            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()\n        else:\n            lgr.log(3, \"Processing provided line\")\n        if line and log_is_callable:\n            # Let it be processed\n            line = log_(assure_unicode(line))\n            if line is not None:\n                # we are working with binary type here\n                line = line.encode()\n        if line:\n            if out_type == 'stdout':\n                self._log_out(assure_unicode(line))\n            elif out_type == 'stderr':\n                self._log_err(line.decode() if PY3 else line,\n                              expected)\n            else:  # pragma: no cover\n                raise RuntimeError(\"must not get here\")\n            return line\n        # it was output already directly but for code to work, return \"\"\n        return binary_type()\n\n    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,\n            expect_stderr=False, expect_fail=False,\n            cwd=None, env=None, shell=None, stdin=None):\n        \"\"\"Runs the command `cmd` using shell.\n\n        In case of dry-mode `cmd` is just added to `commands` and it is\n        actually executed otherwise.\n        Allows for separately logging stdout and stderr  or streaming it to\n        system's stdout or stderr respectively.\n\n        Note: Using a string as `cmd` and shell=True allows for piping,\n              multiple commands, etc., but that implies shlex.split() is not\n              used. This is considered to be a security hazard.\n              So be careful with input.\n\n        Parameters\n        ----------\n        cmd : str, list\n          String (or list) defining the command call.  No shell is used if cmd\n          is specified as a list\n\n        log_stdout: bool, optional\n            If True, stdout is logged. Goes to sys.stdout otherwise.\n\n        log_stderr: bool, optional\n            If True, stderr is logged. Goes to sys.stderr otherwise.\n\n        log_online: bool, optional\n            Either to log as output comes in.  Setting to True is preferable\n            for running user-invoked actions to provide timely output\n\n        expect_stderr: bool, optional\n            Normally, having stderr output is a signal of a problem and thus it\n            gets logged at level 11.  But some utilities, e.g. wget, use\n            stderr for their progress output.  Whenever such output is expected,\n            set it to True and output will be logged at level 9 unless\n            exit status is non-0 (in non-online mode only, in online -- would\n            log at 9)\n\n        expect_fail: bool, optional\n            Normally, if command exits with non-0 status, it is considered an\n            error and logged at level 11 (above DEBUG). But if the call intended\n            for checking routine, such messages are usually not needed, thus\n            it will be logged at level 9.\n\n        cwd : string, optional\n            Directory under which run the command (passed to Popen)\n\n        env : string, optional\n            Custom environment to pass\n\n        shell: bool, optional\n            Run command in a shell.  If not specified, then it runs in a shell\n            only if command is specified as a string (not a list)\n\n        stdin: file descriptor\n            input stream to connect to stdin of the process.\n\n        Returns\n        -------\n        (stdout, stderr)\n\n        Raises\n        ------\n        CommandError\n           if command's exitcode wasn't 0 or None. exitcode is passed to\n           CommandError's `code`-field. Command's stdout and stderr are stored\n           in CommandError's `stdout` and `stderr` fields respectively.\n        \"\"\"\n        outputstream = _get_output_stream(log_stdout, sys.stdout)\n        errstream = _get_output_stream(log_stderr, sys.stderr)\n\n        popen_env = env or self.env\n\n        # TODO: if outputstream is sys.stdout and that one is set to StringIO\n        #       we have to \"shim\" it with something providing fileno().\n        # This happens when we do not swallow outputs, while allowing nosetest's\n        # StringIO to be provided as stdout, crashing the Popen requiring\n        # fileno().  In out swallow_outputs, we just use temporary files\n        # to overcome this problem.\n        # For now necessary test code should be wrapped into swallow_outputs cm\n        # to avoid the problem\n        log_msgs = [\"Running: %s\"]\n        log_args = [cmd]\n        if self.log_cwd:\n            log_msgs += ['cwd=%r']\n            log_args += [cwd or self.cwd]\n        if self.log_stdin:\n            log_msgs += ['stdin=%r']\n            log_args += [stdin]\n        log_env = self.log_env\n        if log_env and popen_env:\n            log_msgs += [\"env=%r\"]\n            log_args.append(\n                popen_env if log_env is True\n                else {k: popen_env[k] for k in log_env if k in popen_env}\n            )\n        log_msg = '\\n'.join(log_msgs)\n        self.log(log_msg, *log_args)\n\n        if self.protocol.do_execute_ext_commands:\n\n            if shell is None:\n                shell = isinstance(cmd, string_types)\n\n            if self.protocol.records_ext_commands:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    shlex.split(cmd, posix=not on_windows)\n                    if isinstance(cmd, string_types)\n                    else cmd)\n            try:\n                proc = subprocess.Popen(cmd,\n                                        stdout=outputstream,\n                                        stderr=errstream,\n                                        shell=shell,\n                                        cwd=cwd or self.cwd,\n                                        env=popen_env,\n                                        stdin=stdin)\n\n            except Exception as e:\n                prot_exc = e\n                lgr.log(11, \"Failed to start %r%r: %s\" %\n                        (cmd, \" under %r\" % cwd if cwd else '', exc_str(e)))\n                raise\n\n            finally:\n                if self.protocol.records_ext_commands:\n                    self.protocol.end_section(prot_id, prot_exc)\n\n            try:\n                if log_online:\n                    out = self._get_output_online(proc,\n                                                  log_stdout, log_stderr,\n                                                  outputstream, errstream,\n                                                  expect_stderr=expect_stderr,\n                                                  expect_fail=expect_fail)\n                else:\n                    out = proc.communicate()\n\n                if PY3:\n                    # Decoding was delayed to this point\n                    def decode_if_not_None(x):\n                        return \"\" if x is None else binary_type.decode(x)\n                    # TODO: check if we can avoid PY3 specific here\n                    out = tuple(map(decode_if_not_None, out))\n\n                status = proc.poll()\n\n                # needs to be done after we know status\n                if not log_online:\n                    self._log_out(out[0])\n                    if status not in [0, None]:\n                        self._log_err(out[1], expected=expect_fail)\n                    else:\n                        # as directed\n                        self._log_err(out[1], expected=expect_stderr)\n\n                if status not in [0, None]:\n                    msg = \"Failed to run %r%s. Exit code=%d. out=%s err=%s\" \\\n                        % (cmd, \" under %r\" % (cwd or self.cwd), status, out[0], out[1])\n                    lgr.log(9 if expect_fail else 11, msg)\n                    raise CommandError(str(cmd), msg, status, out[0], out[1])\n                else:\n                    self.log(\"Finished running %r with status %s\" % (cmd, status),\n                             level=8)\n            finally:\n                # Those streams are for us to close if we asked for a PIPE\n                # TODO -- assure closing the files import pdb; pdb.set_trace()\n                _cleanup_output(outputstream, proc.stdout)\n                _cleanup_output(errstream, proc.stderr)\n\n        else:\n            if self.protocol.records_ext_commands:\n                self.protocol.add_section(shlex.split(cmd,\n                                                      posix=not on_windows)\n                                          if isinstance(cmd, string_types)\n                                          else cmd, None)\n            out = (\"DRY\", \"DRY\")\n\n        return out\n\n    def call(self, f, *args, **kwargs):\n        \"\"\"Helper to unify collection of logging all \"dry\" actions.\n\n        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with\n        its arguments to `commands` otherwise.\n\n        Parameters\n        ----------\n        f: callable\n        \"\"\"\n        if self.protocol.do_execute_callables:\n            if self.protocol.records_callables:\n                prot_exc = None\n                prot_id = self.protocol.start_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)])\n\n            try:\n                return f(*args, **kwargs)\n            except Exception as e:\n                prot_exc = e\n                raise\n            finally:\n                if self.protocol.records_callables:\n                    self.protocol.end_section(prot_id, prot_exc)\n        else:\n            if self.protocol.records_callables:\n                self.protocol.add_section(\n                    [str(f), \"args=%s\" % str(args), \"kwargs=%s\" % str(kwargs)],\n                    None)\n\n    def log(self, msg, *args, **kwargs):\n        \"\"\"log helper\n\n        Logs at level 9 by default and adds \"Protocol:\"-prefix in order to\n        log the used protocol.\n        \"\"\"\n        level = kwargs.pop('level', 9)\n        if isinstance(self.protocol, NullProtocol):\n            lgr.log(level, msg, *args, **kwargs)\n        else:\n            if args:\n                msg = msg % args\n            lgr.log(level, \"{%s} %s\" % (\n                self.protocol.__class__.__name__, msg)\n            )\n\n\nclass GitRunner(Runner):\n    \"\"\"\n    Runner to be used to run git and git annex commands\n\n    Overloads the runner class to check & update GIT_DIR and\n    GIT_WORK_TREE environment variables set to the absolute path\n    if is defined and is relative path\n    \"\"\"\n    _GIT_PATH = None\n\n    @borrowdoc(Runner)\n    def __init__(self, *args, **kwargs):\n        super(GitRunner, self).__init__(*args, **kwargs)\n        self._check_git_path()\n\n    @staticmethod\n    def _check_git_path():\n        \"\"\"If using bundled git-annex, we would like to use bundled with it git\n\n        Thus we will store _GIT_PATH a path to git in the same directory as annex\n        if found.  If it is empty (but not None), we do nothing\n        \"\"\"\n        if GitRunner._GIT_PATH is None:\n            from distutils.spawn import find_executable\n            # with all the nesting of config and this runner, cannot use our\n            # cfg here, so will resort to dark magic of environment options\n            if (os.environ.get('DATALAD_USE_DEFAULT_GIT', '0').lower()\n                    in ('1', 'on', 'true', 'yes')):\n                git_fpath = find_executable(\"git\")\n                if git_fpath:\n                    GitRunner._GIT_PATH = ''\n                    lgr.log(9, \"Will use default git %s\", git_fpath)\n                    return  # we are done - there is a default git avail.\n                # if not -- we will look for a bundled one\n\n            annex_fpath = find_executable(\"git-annex\")\n            if not annex_fpath:\n                # not sure how to live further anyways! ;)\n                alongside = False\n            else:\n                annex_path = os.path.dirname(os.path.realpath(annex_fpath))\n                if on_windows:\n                    # just bundled installations so git should be taken from annex\n                    alongside = True\n                else:\n                    alongside = os.path.lexists(os.path.join(annex_path, 'git'))\n            GitRunner._GIT_PATH = annex_path if alongside else ''\n            lgr.log(9, \"Will use git under %r (no adjustments to PATH if empty \"\n                       \"string)\", GitRunner._GIT_PATH)\n            assert(GitRunner._GIT_PATH is not None)  # we made the decision!\n\n    @staticmethod\n    def get_git_environ_adjusted(env=None):\n        \"\"\"\n        Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined\n        \"\"\"\n        # if env set copy else get os environment\n        git_env = env.copy() if env else os.environ.copy()\n        if GitRunner._GIT_PATH:\n            git_env['PATH'] = pathsep.join([GitRunner._GIT_PATH, git_env['PATH']]) \\\n                if 'PATH' in git_env \\\n                else GitRunner._GIT_PATH\n\n        for varstring in ['GIT_DIR', 'GIT_WORK_TREE']:\n            var = git_env.get(varstring)\n            if var:                                    # if env variable set\n                if not isabs(var):                     # and it's a relative path\n                    git_env[varstring] = abspath(var)  # to absolute path\n                    lgr.log(9, \"Updated %s to %s\", varstring, git_env[varstring])\n\n        if 'GIT_SSH_COMMAND' not in git_env:\n            git_env['GIT_SSH_COMMAND'] = GIT_SSH_COMMAND\n\n        return git_env\n\n    def run(self, cmd, env=None, *args, **kwargs):\n        return super(GitRunner, self).run(\n            cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)\n\n\n# ####\n# Preserve from previous version\n# TODO: document intention\n# ####\n# this one might get under Runner for better output/control\ndef link_file_load(src, dst, dry_run=False):\n    \"\"\"Just a little helper to hardlink files's load\n    \"\"\"\n    dst_dir = os.path.dirname(dst)\n    if not os.path.exists(dst_dir):\n        os.makedirs(dst_dir)\n    if os.path.lexists(dst):\n        lgr.log(9, \"Destination file %(dst)s exists. Removing it first\", locals())\n        # TODO: how would it interact with git/git-annex\n        os.unlink(dst)\n    lgr.log(9, \"Hardlinking %(src)s under %(dst)s\", locals())\n    src_realpath = os.path.realpath(src)\n\n    try:\n        os.link(src_realpath, dst)\n    except AttributeError as e:\n        lgr.warn(\"Linking of %s failed (%s), copying file\" % (src, e))\n        shutil.copyfile(src_realpath, dst)\n        shutil.copystat(src_realpath, dst)\n    else:\n        lgr.log(2, \"Hardlinking finished\")\n\n\ndef get_runner(*args, **kwargs):\n    # needs local import, because the ConfigManager itself needs the runner\n    from . import cfg\n    # TODO:  this is all crawl specific -- should be moved away\n    if cfg.obtain('datalad.crawl.dryrun', default=False):\n        kwargs = kwargs.copy()\n        kwargs['protocol'] = DryRunProtocol()\n    return Runner(*args, **kwargs)\n"}
A user will ask you to solve a task. You should first draft your thinking process (inner monologue). Then, generate the solution.

Your response format must follow the template below:
<think>
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct solution.
</think>
<solution>
Final solution presented to the user.
</solution>
system
We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[ISSUE]
**Title:** Missing newline character at the end of processed output lines causes assertion failure

**Description:**
When processing output lines, the lines returned do not consistently end with a newline character (`\n`). This leads to unexpected behavior in downstream processes that rely on each line being properly terminated.

**Example:**
```python
def process_output():
    out = process_remaining_output(b's\xd0\xbf')
    print(out)

def process_remaining_output(data):
    # Expected to ensure each line ends with a newline
    return data  # Missing newline characters

process_output()
```
In the example above, the processed output `b's\xd0\xbf'` lacks the trailing newline characters, whereas the expected output should be `b's\n\xd0\xbf\n'`.

**Expected Behavior:**
Each line returned after processing should end with a newline character (`\n`). For example, given the input `b's\xd0\xbf'`, the output should be `b's\n\xd0\xbf\n'`.

**Actual Behavior:**
The processed output does not include the required newline characters, resulting in `b's\xd0\xbf'` instead of the expected `b's\n\xd0\xbf\n'`. This discrepancy causes assertion failures in tests that validate the presence of newline characters at the end of each line.

[/ISSUE]
--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.

--- BEGIN FILE ---
```
### datalad/cmd.py
# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 noet:
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the datalad package for the
#   copyright and license terms.
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""
Wrapper for command and function calls, allowing for dry runs and output handling

"""


import subprocess
import sys
import logging
import os
import shutil
import shlex
import atexit
import functools
import tempfile

from collections import OrderedDict
from six import PY3, PY2
from six import string_types, binary_type, text_type
from os.path import abspath, isabs, pathsep, exists

from .consts import GIT_SSH_COMMAND
from .dochelpers import exc_str
from .support.exceptions import CommandError
from .support.protocol import NullProtocol, DryRunProtocol, \
    ExecutionTimeProtocol, ExecutionTimeExternalsProtocol
from .utils import on_windows, get_tempfile_kwargs, assure_unicode
from .dochelpers import borrowdoc

lgr = logging.getLogger('datalad.cmd')

# In python3 to split byte stream on newline, it must be bytes
linesep_bytes = os.linesep.encode()

_TEMP_std = sys.stdout, sys.stderr
# To be used in the temp file name to distinguish the ones we create
# in Runner so we take care about their removal, in contrast to those
# which might be created outside and passed into Runner
_MAGICAL_OUTPUT_MARKER = "_runneroutput_"

if PY2:
    # TODO apparently there is a recommended substitution for Python2
    # which is a backported implementation of python3 subprocess
    # https://pypi.python.org/pypi/subprocess32/
    file_class = file
else:
    from io import IOBase as file_class


def _decide_to_log(v):
    """Hacky workaround for now so we could specify per each which to
    log online and which to the log"""
    if isinstance(v, bool) or callable(v):
        return v
    elif v in {'online'}:
        return True
    elif v in {'offline'}:
        return False
    else:
        raise ValueError("can be bool, callable, 'online' or 'offline'")


def _get_output_stream(log_std, false_value):
    """Helper to prepare output stream for Popen and use file for 'offline'

    Necessary to avoid lockdowns when both stdout and stderr are pipes
    """
    if log_std:
        if log_std == 'offline':
            # we will open a temporary file

            tf = tempfile.mktemp(
                **get_tempfile_kwargs({}, prefix=_MAGICAL_OUTPUT_MARKER)
            )
            return open(tf, 'w')  # XXX PY3 should be 'b' may be?
        else:
            return subprocess.PIPE
    else:
        return false_value


def _cleanup_output(stream, std):
    if isinstance(stream, file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:
        if not stream.closed:
            stream.close()
        if exists(stream.name):
            os.unlink(stream.name)
    elif stream == subprocess.PIPE:
        std.close()


class Runner(object):
    """Provides a wrapper for calling functions and commands.

    An object of this class provides a methods that calls shell commands or
    python functions, allowing for protocolling the calls and output handling.

    Outputs (stdout and stderr) can be either logged or streamed to system's
    stdout/stderr during execution.
    This can be enabled or disabled for both of them independently.
    Additionally, a protocol object can be a used with the Runner. Such a
    protocol has to implement datalad.support.protocol.ProtocolInterface, is
    able to record calls and allows for dry runs.
    """

    __slots__ = ['commands', 'dry', 'cwd', 'env', 'protocol',
                 '_log_opts']

    def __init__(self, cwd=None, env=None, protocol=None, log_outputs=None):
        """
        Parameters
        ----------
        cwd: string, optional
             Base current working directory for commands.  Could be overridden
             per run call via cwd option
        env: dict, optional
             Custom environment to use for calls. Could be overridden per run
             call via env option
        protocol: ProtocolInterface
             Protocol object to write to.
        log_outputs : bool, optional
             Switch to instruct either outputs should be logged or not.  If not
             set (default), config 'datalad.log.outputs' would be consulted
        """

        self.cwd = cwd
        self.env = env
        if protocol is None:
            # TODO: config cmd.protocol = null
            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')
            protocol = {
                'externals-time': ExecutionTimeExternalsProtocol,
                'time': ExecutionTimeProtocol,
                'null': NullProtocol
            }[protocol_str]()
            if protocol_str != 'null':
                # we need to dump it into a file at the end
                # TODO: config cmd.protocol_prefix = protocol
                filename = '%s-%s.log' % (
                    os.environ.get('DATALAD_CMD_PROTOCOL_PREFIX', 'protocol'),
                    id(self)
                )
                atexit.register(functools.partial(protocol.write_to_file, filename))

        self.protocol = protocol
        # Various options for logging
        self._log_opts = {}
        # we don't know yet either we need ot log every output or not
        if log_outputs is not None:
            self._log_opts['outputs'] = log_outputs

    def __call__(self, cmd, *args, **kwargs):
        """Convenience method

        This will call run() or call() depending on the kind of `cmd`.
        If `cmd` is a string it is interpreted as the to be executed command.
        Otherwise it is expected to be a callable.
        Any other argument is passed to the respective method.

        Parameters
        ----------
        cmd: str or callable
           command string to be executed via shell or callable to be called.

        `*args`:
        `**kwargs`:
           see Runner.run() and Runner.call() for available arguments.

        Raises
        ------
        TypeError
          if cmd is neither a string nor a callable.
        """

        if isinstance(cmd, string_types) or isinstance(cmd, list):
            return self.run(cmd, *args, **kwargs)
        elif callable(cmd):
            return self.call(cmd, *args, **kwargs)
        else:
            raise TypeError("Argument 'command' is neither a string, "
                            "nor a list nor a callable.")

    def _opt_env_adapter(v):
        """If value is a string, split by ,"""
        if v:
            if v.isdigit():
                log_env = bool(int(v))
            else:
                log_env = v.split(',')
            return log_env
        else:
            return False

    _LOG_OPTS_ADAPTERS = OrderedDict([
        ('outputs', None),
        ('cwd', None),
        ('env', _opt_env_adapter),
        ('stdin', None),
    ])

    def _get_log_setting(self, opt, default=False):
        try:
            return self._log_opts[opt]
        except KeyError:
            try:
                from . import cfg
            except ImportError:
                return default
            adapter = self._LOG_OPTS_ADAPTERS.get(opt, None)
            self._log_opts[opt] = \
                (cfg.getbool if not adapter else cfg.get_value)(
                    'datalad.log.cmd', opt, default=default)
            if adapter:
                self._log_opts[opt] = adapter(self._log_opts[opt])
            return self._log_opts[opt]

    @property
    def log_outputs(self):
        return self._get_log_setting('outputs')

    @property
    def log_cwd(self):
        return self._get_log_setting('cwd')

    @property
    def log_stdin(self):
        return self._get_log_setting('stdin')

    @property
    def log_env(self):
        return self._get_log_setting('env')

    # Two helpers to encapsulate formatting/output
    def _log_out(self, line):
        if line and self.log_outputs:
            self.log("stdout| " + line.rstrip('\n'))

    def _log_err(self, line, expected=False):
        if line and self.log_outputs:
            self.log("stderr| " + line.rstrip('\n'),
                     level={True: 9,
                            False: 11}[expected])

    def _get_output_online(self, proc,
                           log_stdout, log_stderr,
                           outputstream, errstream,
                           expect_stderr=False, expect_fail=False):
        """

        If log_stdout or log_stderr are callables, they will be given a read
        line to be processed, and return processed result.  So if they need to
        'swallow' the line from being logged, should just return None

        Parameters
        ----------
        proc
        log_stdout: bool or callable or 'online' or 'offline'
        log_stderr: : bool or callable or 'online' or 'offline'
          If any of those 'offline', we would call proc.communicate at the
          end to grab possibly outstanding output from it
        expect_stderr
        expect_fail

        Returns
        -------

        """
        stdout, stderr = binary_type(), binary_type()

        log_stdout_ = _decide_to_log(log_stdout)
        log_stderr_ = _decide_to_log(log_stderr)
        log_stdout_is_callable = callable(log_stdout_)
        log_stderr_is_callable = callable(log_stderr_)

        # arguments to be passed into _process_one_line
        stdout_args = (
                'stdout',
                proc, log_stdout_, log_stdout_is_callable
        )
        stderr_args = (
                'stderr',
                proc, log_stderr_, log_stderr_is_callable,
                expect_stderr or expect_fail
        )

        while proc.poll() is None:
            # see for a possibly useful approach to processing output
            # in another thread http://codereview.stackexchange.com/a/17959
            # current problem is that if there is no output on stderr
            # it stalls
            if log_stdout_:
                stdout += self._process_one_line(*stdout_args)
            if log_stderr_:
                stderr += self._process_one_line(*stderr_args)

        # Handle possible remaining output
        stdout_, stderr_ = proc.communicate()
        # ??? should we condition it on log_stdout in {'offline'} ???
        stdout += self._process_remaining_output(outputstream, stdout_, *stdout_args)
        stderr += self._process_remaining_output(errstream, stderr_, *stderr_args)

        return stdout, stderr

    def _process_remaining_output(self, stream, out_, *pargs):
        """Helper to process output which might have been obtained from popen or
        should be loaded from file"""
        out = binary_type()
        if isinstance(stream,
                      file_class) and _MAGICAL_OUTPUT_MARKER in stream.name:
            assert out_ is None, "should have gone into a file"
            if not stream.closed:
                stream.close()
            with open(stream.name, 'rb') as f:
                for line in f:
                    out += self._process_one_line(*pargs, line=line)
        else:
            if out_:
                # resolving a once in a while failing test #2185
                if isinstance(out_, text_type):
                    out_ = out_.encode('utf-8')
                for line in out_.split(linesep_bytes):
                    out += self._process_one_line(*pargs, line=line)
        return out

    def _process_one_line(self, out_type, proc, log_, log_is_callable,
                          expected=False, line=None):
        if line is None:
            lgr.log(3, "Reading line from %s", out_type)
            line = {'stdout': proc.stdout, 'stderr': proc.stderr}[out_type].readline()
        else:
            lgr.log(3, "Processing provided line")
        if line and log_is_callable:
            # Let it be processed
            line = log_(assure_unicode(line))
            if line is not None:
                # we are working with binary type here
                line = line.encode()
        if line:
            if out_type == 'stdout':
                self._log_out(assure_unicode(line))
            elif out_type == 'stderr':
                self._log_err(line.decode() if PY3 else line,
                              expected)
            else:  # pragma: no cover
                raise RuntimeError("must not get here")
            return line
        # it was output already directly but for code to work, return ""
        return binary_type()

    def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,
            expect_stderr=False, expect_fail=False,
            cwd=None, env=None, shell=None, stdin=None):
        """Runs the command `cmd` using shell.

        In case of dry-mode `cmd` is just added to `commands` and it is
        actually executed otherwise.
        Allows for separately logging stdout and stderr  or streaming it to
        system's stdout or stderr respectively.

        Note: Using a string as `cmd` and shell=True allows for piping,
              multiple commands, etc., but that implies shlex.split() is not
              used. This is considered to be a security hazard.
              So be careful with input.

        Parameters
        ----------
        cmd : str, list
          String (or list) defining the command call.  No shell is used if cmd
          is specified as a list

        log_stdout: bool, optional
            If True, stdout is logged. Goes to sys.stdout otherwise.

        log_stderr: bool, optional
            If True, stderr is logged. Goes to sys.stderr otherwise.

        log_online: bool, optional
            Either to log as output comes in.  Setting to True is preferable
            for running user-invoked actions to provide timely output

        expect_stderr: bool, optional
            Normally, having stderr output is a signal of a problem and thus it
            gets logged at level 11.  But some utilities, e.g. wget, use
            stderr for their progress output.  Whenever such output is expected,
            set it to True and output will be logged at level 9 unless
            exit status is non-0 (in non-online mode only, in online -- would
            log at 9)

        expect_fail: bool, optional
            Normally, if command exits with non-0 status, it is considered an
            error and logged at level 11 (above DEBUG). But if the call intended
            for checking routine, such messages are usually not needed, thus
            it will be logged at level 9.

        cwd : string, optional
            Directory under which run the command (passed to Popen)

        env : string, optional
            Custom environment to pass

        shell: bool, optional
            Run command in a shell.  If not specified, then it runs in a shell
            only if command is specified as a string (not a list)

        stdin: file descriptor
            input stream to connect to stdin of the process.

        Returns
        -------
        (stdout, stderr)

        Raises
        ------
        CommandError
           if command's exitcode wasn't 0 or None. exitcode is passed to
           CommandError's `code`-field. Command's stdout and stderr are stored
           in CommandError's `stdout` and `stderr` fields respectively.
        """
        outputstream = _get_output_stream(log_stdout, sys.stdout)
        errstream = _get_output_stream(log_stderr, sys.stderr)

        popen_env = env or self.env

        # TODO: if outputstream is sys.stdout and that one is set to StringIO
        #       we have to "shim" it with something providing fileno().
        # This happens when we do not swallow outputs, while allowing nosetest's
        # StringIO to be provided as stdout, crashing the Popen requiring
        # fileno().  In out swallow_outputs, we just use temporary files
        # to overcome this problem.
        # For now necessary test code should be wrapped into swallow_outputs cm
        # to avoid the problem
        log_msgs = ["Running: %s"]
        log_args = [cmd]
        if self.log_cwd:
            log_msgs += ['cwd=%r']
            log_args += [cwd or self.cwd]
        if self.log_stdin:
            log_msgs += ['stdin=%r']
            log_args += [stdin]
        log_env = self.log_env
        if log_env and popen_env:
            log_msgs += ["env=%r"]
            log_args.append(
                popen_env if log_env is True
                else {k: popen_env[k] for k in log_env if k in popen_env}
            )
        log_msg = '\n'.join(log_msgs)
        self.log(log_msg, *log_args)

        if self.protocol.do_execute_ext_commands:

            if shell is None:
                shell = isinstance(cmd, string_types)

            if self.protocol.records_ext_commands:
                prot_exc = None
                prot_id = self.protocol.start_section(
                    shlex.split(cmd, posix=not on_windows)
                    if isinstance(cmd, string_types)
                    else cmd)
            try:
                proc = subprocess.Popen(cmd,
                                        stdout=outputstream,
                                        stderr=errstream,
                                        shell=shell,
                                        cwd=cwd or self.cwd,
                                        env=popen_env,
                                        stdin=stdin)

            except Exception as e:
                prot_exc = e
                lgr.log(11, "Failed to start %r%r: %s" %
                        (cmd, " under %r" % cwd if cwd else '', exc_str(e)))
                raise

            finally:
                if self.protocol.records_ext_commands:
                    self.protocol.end_section(prot_id, prot_exc)

            try:
                if log_online:
                    out = self._get_output_online(proc,
                                                  log_stdout, log_stderr,
                                                  outputstream, errstream,
                                                  expect_stderr=expect_stderr,
                                                  expect_fail=expect_fail)
                else:
                    out = proc.communicate()

                if PY3:
                    # Decoding was delayed to this point
                    def decode_if_not_None(x):
                        return "" if x is None else binary_type.decode(x)
                    # TODO: check if we can avoid PY3 specific here
                    out = tuple(map(decode_if_not_None, out))

                status = proc.poll()

                # needs to be done after we know status
                if not log_online:
                    self._log_out(out[0])
                    if status not in [0, None]:
                        self._log_err(out[1], expected=expect_fail)
                    else:
                        # as directed
                        self._log_err(out[1], expected=expect_stderr)

                if status not in [0, None]:
                    msg = "Failed to run %r%s. Exit code=%d. out=%s err=%s" \
                        % (cmd, " under %r" % (cwd or self.cwd), status, out[0], out[1])
                    lgr.log(9 if expect_fail else 11, msg)
                    raise CommandError(str(cmd), msg, status, out[0], out[1])
                else:
                    self.log("Finished running %r with status %s" % (cmd, status),
                             level=8)
            finally:
                # Those streams are for us to close if we asked for a PIPE
                # TODO -- assure closing the files import pdb; pdb.set_trace()
                _cleanup_output(outputstream, proc.stdout)
                _cleanup_output(errstream, proc.stderr)

        else:
            if self.protocol.records_ext_commands:
                self.protocol.add_section(shlex.split(cmd,
                                                      posix=not on_windows)
                                          if isinstance(cmd, string_types)
                                          else cmd, None)
            out = ("DRY", "DRY")

        return out

    def call(self, f, *args, **kwargs):
        """Helper to unify collection of logging all "dry" actions.

        Calls `f` if `Runner`-object is not in dry-mode. Adds `f` along with
        its arguments to `commands` otherwise.

        Parameters
        ----------
        f: callable
        """
        if self.protocol.do_execute_callables:
            if self.protocol.records_callables:
                prot_exc = None
                prot_id = self.protocol.start_section(
                    [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)])

            try:
                return f(*args, **kwargs)
            except Exception as e:
                prot_exc = e
                raise
            finally:
                if self.protocol.records_callables:
                    self.protocol.end_section(prot_id, prot_exc)
        else:
            if self.protocol.records_callables:
                self.protocol.add_section(
                    [str(f), "args=%s" % str(args), "kwargs=%s" % str(kwargs)],
                    None)

    def log(self, msg, *args, **kwargs):
        """log helper

        Logs at level 9 by default and adds "Protocol:"-prefix in order to
        log the used protocol.
        """
        level = kwargs.pop('level', 9)
        if isinstance(self.protocol, NullProtocol):
            lgr.log(level, msg, *args, **kwargs)
        else:
            if args:
                msg = msg % args
            lgr.log(level, "{%s} %s" % (
                self.protocol.__class__.__name__, msg)
            )


class GitRunner(Runner):
    """
    Runner to be used to run git and git annex commands

    Overloads the runner class to check & update GIT_DIR and
    GIT_WORK_TREE environment variables set to the absolute path
    if is defined and is relative path
    """
    _GIT_PATH = None

    @borrowdoc(Runner)
    def __init__(self, *args, **kwargs):
        super(GitRunner, self).__init__(*args, **kwargs)
        self._check_git_path()

    @staticmethod
    def _check_git_path():
        """If using bundled git-annex, we would like to use bundled with it git

        Thus we will store _GIT_PATH a path to git in the same directory as annex
        if found.  If it is empty (but not None), we do nothing
        """
        if GitRunner._GIT_PATH is None:
            from distutils.spawn import find_executable
            # with all the nesting of config and this runner, cannot use our
            # cfg here, so will resort to dark magic of environment options
            if (os.environ.get('DATALAD_USE_DEFAULT_GIT', '0').lower()
                    in ('1', 'on', 'true', 'yes')):
                git_fpath = find_executable("git")
                if git_fpath:
                    GitRunner._GIT_PATH = ''
                    lgr.log(9, "Will use default git %s", git_fpath)
                    return  # we are done - there is a default git avail.
                # if not -- we will look for a bundled one

            annex_fpath = find_executable("git-annex")
            if not annex_fpath:
                # not sure how to live further anyways! ;)
                alongside = False
            else:
                annex_path = os.path.dirname(os.path.realpath(annex_fpath))
                if on_windows:
                    # just bundled installations so git should be taken from annex
                    alongside = True
                else:
                    alongside = os.path.lexists(os.path.join(annex_path, 'git'))
            GitRunner._GIT_PATH = annex_path if alongside else ''
            lgr.log(9, "Will use git under %r (no adjustments to PATH if empty "
                       "string)", GitRunner._GIT_PATH)
            assert(GitRunner._GIT_PATH is not None)  # we made the decision!

    @staticmethod
    def get_git_environ_adjusted(env=None):
        """
        Replaces GIT_DIR and GIT_WORK_TREE with absolute paths if relative path and defined
        """
        # if env set copy else get os environment
        git_env = env.copy() if env else os.environ.copy()
        if GitRunner._GIT_PATH:
            git_env['PATH'] = pathsep.join([GitRunner._GIT_PATH, git_env['PATH']]) \
                if 'PATH' in git_env \
                else GitRunner._GIT_PATH

        for varstring in ['GIT_DIR', 'GIT_WORK_TREE']:
            var = git_env.get(varstring)
            if var:                                    # if env variable set
                if not isabs(var):                     # and it's a relative path
                    git_env[varstring] = abspath(var)  # to absolute path
                    lgr.log(9, "Updated %s to %s", varstring, git_env[varstring])

        if 'GIT_SSH_COMMAND' not in git_env:
            git_env['GIT_SSH_COMMAND'] = GIT_SSH_COMMAND

        return git_env

    def run(self, cmd, env=None, *args, **kwargs):
        return super(GitRunner, self).run(
            cmd, env=self.get_git_environ_adjusted(env), *args, **kwargs)


# ####
# Preserve from previous version
# TODO: document intention
# ####
# this one might get under Runner for better output/control
def link_file_load(src, dst, dry_run=False):
    """Just a little helper to hardlink files's load
    """
    dst_dir = os.path.dirname(dst)
    if not os.path.exists(dst_dir):
        os.makedirs(dst_dir)
    if os.path.lexists(dst):
        lgr.log(9, "Destination file %(dst)s exists. Removing it first", locals())
        # TODO: how would it interact with git/git-annex
        os.unlink(dst)
    lgr.log(9, "Hardlinking %(src)s under %(dst)s", locals())
    src_realpath = os.path.realpath(src)

    try:
        os.link(src_realpath, dst)
    except AttributeError as e:
        lgr.warn("Linking of %s failed (%s), copying file" % (src, e))
        shutil.copyfile(src_realpath, dst)
        shutil.copystat(src_realpath, dst)
    else:
        lgr.log(2, "Hardlinking finished")


def get_runner(*args, **kwargs):
    # needs local import, because the ConfigManager itself needs the runner
    from . import cfg
    # TODO:  this is all crawl specific -- should be moved away
    if cfg.obtain('datalad.crawl.dryrun', default=False):
        kwargs = kwargs.copy()
        kwargs['protocol'] = DryRunProtocol()
    return Runner(*args, **kwargs)

```
--- END FILE ---

Please first localize the bug based on the issue statement, and then generate *SEARCH/REPLACE* edits to fix the issue.

Every *SEARCH/REPLACE* edit must use this format:
1. The file path
2. The start of search block: <<<<<<< SEARCH
3. A contiguous chunk of lines to search for in the existing source code
4. The dividing line: =======
5. The lines to replace into the source code
6. The end of the replace block: >>>>>>> REPLACE

Here is an example:

```python
### mathweb/flask/app.py
<<<<<<< SEARCH
from flask import Flask
=======
import math
from flask import Flask
>>>>>>> REPLACE
```
</solution>

Please note that the *SEARCH/REPLACE* edit REQUIRES PROPER INDENTATION. If you would like to add the line '        print(x)', you must fully write that out, with all those spaces before the code!
Wrap each *SEARCH/REPLACE* edit in a code block as shown in the example above. If you have multiple *SEARCH/REPLACE* edits, use a separate code block for each one.
user