code: def _run_with_output_on_failure(self, cmd, non_blocking=False,
                                    ignore_errors=False,
                                    max_wait_sec=365 * 24 * 3600,
                                    check_interval=0.2) -> str:
      if not self._can_run:
        assert False, "Using .run before initialization finished"
      if '\n' in cmd:
        assert False, "Don't support multi-line for run2"
      cmd = cmd.strip()
      if cmd.startswith('
        return ''
      self.run_counter += 1
      self.log("tmux> %s", cmd)
      self._cmd = cmd
      self._cmd_fn = f'{self.remote_scratch}/{self.run_counter}.cmd'
      self._status_fn = f'{self.remote_scratch}/{self.run_counter}.status'
      self._out_fn = f'{self.remote_scratch}/{self.run_counter}.out'
      cmd = util.shell_strip_comment(cmd)
      assert '&' not in cmd, f"cmd {cmd} contains &, that breaks things"
      self.file_write(self._cmd_fn, cmd + '\n')
      modified_cmd = f'{cmd} > >(tee -a {self._out_fn}) 2> >(tee -a {self._out_fn} >&2); echo $? > {self._status_fn}'
      modified_cmd = shlex.quote(modified_cmd)
      start_time = time.time()
      tmux_window = self.tmux_session + ':' + str(self.tmux_window_id)
      tmux_cmd = f"tmux send-keys -t {tmux_window} {modified_cmd} Enter"
      self._run_raw(tmux_cmd, ignore_errors=ignore_errors)
      if non_blocking:
        return 0
      if not self.wait_for_file(self._status_fn, max_wait_sec=60):
        self.log(f"Retrying waiting for {self._status_fn}")
      elapsed_time = time.time() - start_time
      while not self.exists(self._status_fn) and elapsed_time < max_wait_sec:
        self.log(f"Still waiting for {cmd}")
        self.wait_for_file(self._status_fn, max_wait_sec=60)
        elapsed_time = time.time() - start_time
      contents = self.read(self._status_fn)
      if len(contents) == 0:
        time.sleep(check_interval)
        contents = self.read(self._status_fn)
      status = int(contents.strip())
      self.last_status = status
      if status != 0:
        extra_msg = '(ignoring error)' if ignore_errors else '(failing)'
        self.log(
          f"Start failing output {extra_msg}: \n{'*'*80}\n\n '{self.read(self._out_fn)}'")
        self.log(f"\n{'*'*80}\nEnd failing output")
        if not ignore_errors:
          raise RuntimeError(f"Command {cmd} returned status {status}")
        else:
          self.log(f"Warning: command {cmd} returned status {status}")
      return self.read(self._out_fn)