We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Req to deal with SLURM socket errors more patiently
### Summary

At end of issue #2693 @effigies noted that the error that @dalejn was experiencing was due to the SLURM master throwing an error when it was polled with squeue, possibly because it was busy. After some further testing, we now believe that the NIH HPC SLURM master will throw this error at least once a day even with a modest polling interval. 

We would like to request a patch such that if NiPype receives any kind of timeout error (we've seen a few different kinds) from squeue, that it politely waits and tries again.

### Actual behavior
```
RuntimeError: Command:
squeue -j 9448406
Standard output:

Standard error:
slurm_load_jobs error: Socket timed out on send/recv operation
Return code: 1
```
or
```
The batch system is not available at the moment.
```
and NiPype exits
### Requested behavior
```
squeue is busy, will try again
```
And NiPype does _not_exit

### Platform details:
```
(NiPypeUpdate) [zhoud4@felix ETPB]$ python -c "import nipype; from pprint import pprint; pprint(nipype.get_info())"
{'commit_hash': 'ec7457c23',
 'commit_source': 'installation',
 'networkx_version': '2.2',
 'nibabel_version': '2.3.1',
 'nipype_version': '1.1.3',
 'numpy_version': '1.15.3',
 'pkg_path': '/data/zhoud4/python/envs/NiPypeUpdate/lib/python3.5/site-packages/nipype',
 'scipy_version': '1.1.0',
 'sys_executable': '/data/zhoud4/python/envs/NiPypeUpdate/bin/python',
 'sys_platform': 'linux',
 'sys_version': '3.5.4 | packaged by conda-forge | (default, Aug 10 2017, '
                '01:38:41) \n'
                '[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]',
 'traits_version': '4.6.0'}
(NiPypeUpdate) [zhoud4@felix ETPB]$
(NiPypeUpdate) [zhoud4@biowulf ETPB]$ sinfo -V
slurm 17.02.9
(NiPypeUpdate) [zhoud4@biowulf ETPB]$ 
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nipype/pipeline/plugins/slurm.py`
Content:
```
1 '''
2 Created on Aug 2, 2013
3 
4 @author: chadcumba
5 
6 Parallel workflow execution with SLURM
7 '''
8 from __future__ import (print_function, division, unicode_literals,
9                         absolute_import)
10 from builtins import open
11 
12 import os
13 import re
14 from time import sleep
15 
16 from ... import logging
17 from ...interfaces.base import CommandLine
18 from .base import SGELikeBatchManagerBase, logger
19 
20 iflogger = logging.getLogger('nipype.interface')
21 
22 
23 class SLURMPlugin(SGELikeBatchManagerBase):
24     '''
25     Execute using SLURM
26 
27     The plugin_args input to run can be used to control the SLURM execution.
28     Currently supported options are:
29 
30     - template : template to use for batch job submission
31 
32     - sbatch_args: arguments to pass prepend to the sbatch call
33 
34 
35     '''
36 
37     def __init__(self, **kwargs):
38 
39         template = "#!/bin/bash"
40 
41         self._retry_timeout = 2
42         self._max_tries = 2
43         self._template = template
44         self._sbatch_args = None
45         self._jobid_re = "Submitted batch job ([0-9]*)"
46 
47         if 'plugin_args' in kwargs and kwargs['plugin_args']:
48             if 'retry_timeout' in kwargs['plugin_args']:
49                 self._retry_timeout = kwargs['plugin_args']['retry_timeout']
50             if 'max_tries' in kwargs['plugin_args']:
51                 self._max_tries = kwargs['plugin_args']['max_tries']
52             if 'jobid_re' in kwargs['plugin_args']:
53                 self._jobid_re = kwargs['plugin_args']['jobid_re']
54             if 'template' in kwargs['plugin_args']:
55                 self._template = kwargs['plugin_args']['template']
56                 if os.path.isfile(self._template):
57                     with open(self._template) as f:
58                         self._template = f.read()
59             if 'sbatch_args' in kwargs['plugin_args']:
60                 self._sbatch_args = kwargs['plugin_args']['sbatch_args']
61         self._pending = {}
62         super(SLURMPlugin, self).__init__(self._template, **kwargs)
63 
64     def _is_pending(self, taskid):
65         try:
66             res = CommandLine(
67                 'squeue',
68                 args=' '.join(['-j', '%s' % taskid]),
69                 resource_monitor=False,
70                 terminal_output='allatonce').run()
71             return res.runtime.stdout.find(str(taskid)) > -1
72         except RuntimeError as e:
73             if 'Invalid job id' not in str(e):
74                 raise(e)
75             return False
76 
77     def _submit_batchtask(self, scriptfile, node):
78         """
79         This is more or less the _submit_batchtask from sge.py with flipped
80         variable names, different command line switches, and different output
81         formatting/processing
82         """
83         cmd = CommandLine(
84             'sbatch',
85             environ=dict(os.environ),
86             resource_monitor=False,
87             terminal_output='allatonce')
88         path = os.path.dirname(scriptfile)
89 
90         sbatch_args = ''
91         if self._sbatch_args:
92             sbatch_args = self._sbatch_args
93         if 'sbatch_args' in node.plugin_args:
94             if 'overwrite' in node.plugin_args and\
95                node.plugin_args['overwrite']:
96                 sbatch_args = node.plugin_args['sbatch_args']
97             else:
98                 sbatch_args += (" " + node.plugin_args['sbatch_args'])
99         if '-o' not in sbatch_args:
100             sbatch_args = '%s -o %s' % (sbatch_args,
101                                         os.path.join(path, 'slurm-%j.out'))
102         if '-e' not in sbatch_args:
103             sbatch_args = '%s -e %s' % (sbatch_args,
104                                         os.path.join(path, 'slurm-%j.out'))
105         if node._hierarchy:
106             jobname = '.'.join((dict(os.environ)['LOGNAME'], node._hierarchy,
107                                 node._id))
108         else:
109             jobname = '.'.join((dict(os.environ)['LOGNAME'], node._id))
110         jobnameitems = jobname.split('.')
111         jobnameitems.reverse()
112         jobname = '.'.join(jobnameitems)
113         cmd.inputs.args = '%s -J %s %s' % (sbatch_args, jobname, scriptfile)
114         oldlevel = iflogger.level
115         iflogger.setLevel(logging.getLevelName('CRITICAL'))
116         tries = 0
117         while True:
118             try:
119                 result = cmd.run()
120             except Exception as e:
121                 if tries < self._max_tries:
122                     tries += 1
123                     # sleep 2 seconds and try again.
124                     sleep(self._retry_timeout)
125                 else:
126                     iflogger.setLevel(oldlevel)
127                     raise RuntimeError('\n'.join(
128                         (('Could not submit sbatch task'
129                           ' for node %s') % node._id, str(e))))
130             else:
131                 break
132         logger.debug('Ran command ({0})'.format(cmd.cmdline))
133         iflogger.setLevel(oldlevel)
134         # retrieve taskid
135         lines = [line for line in result.runtime.stdout.split('\n') if line]
136         taskid = int(re.match(self._jobid_re, lines[-1]).groups()[0])
137         self._pending[taskid] = node.output_dir()
138         logger.debug('submitted sbatch task: %d for node %s' % (taskid,
139                                                                 node._id))
140         return taskid
141
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nipype/pipeline/plugins/slurm.py b/nipype/pipeline/plugins/slurm.py
--- a/nipype/pipeline/plugins/slurm.py
+++ b/nipype/pipeline/plugins/slurm.py
@@ -70,6 +70,14 @@
                 terminal_output='allatonce').run()
             return res.runtime.stdout.find(str(taskid)) > -1
         except RuntimeError as e:
+            if any(ss in str(e) for ss
+                   in ['Socket timed out', 'not available at the moment']):
+                # do not raise error and allow recheck
+                logger.warning(
+                    "SLURM timeout encountered while checking job status,"
+                    " treating job %d as pending", taskid
+                )
+                return True
             if 'Invalid job id' not in str(e):
                 raise(e)
             return False

{"golden_diff": "diff --git a/nipype/pipeline/plugins/slurm.py b/nipype/pipeline/plugins/slurm.py\n--- a/nipype/pipeline/plugins/slurm.py\n+++ b/nipype/pipeline/plugins/slurm.py\n@@ -70,6 +70,14 @@\n                 terminal_output='allatonce').run()\n             return res.runtime.stdout.find(str(taskid)) > -1\n         except RuntimeError as e:\n+            if any(ss in str(e) for ss\n+                   in ['Socket timed out', 'not available at the moment']):\n+                # do not raise error and allow recheck\n+                logger.warning(\n+                    \"SLURM timeout encountered while checking job status,\"\n+                    \" treating job %d as pending\", taskid\n+                )\n+                return True\n             if 'Invalid job id' not in str(e):\n                 raise(e)\n             return False\n", "issue": "Req to deal with SLURM socket errors more patiently\n### Summary\r\n\r\nAt end of issue #2693 @effigies noted that the error that @dalejn was experiencing was due to the SLURM master throwing an error when it was polled with squeue, possibly because it was busy. After some further testing, we now believe that the NIH HPC SLURM master will throw this error at least once a day even with a modest polling interval. \r\n\r\nWe would like to request a patch such that if NiPype receives any kind of timeout error (we've seen a few different kinds) from squeue, that it politely waits and tries again.\r\n\r\n### Actual behavior\r\n```\r\nRuntimeError: Command:\r\nsqueue -j 9448406\r\nStandard output:\r\n\r\nStandard error:\r\nslurm_load_jobs error: Socket timed out on send/recv operation\r\nReturn code: 1\r\n```\r\nor\r\n```\r\nThe batch system is not available at the moment.\r\n```\r\nand NiPype exits\r\n### Requested behavior\r\n```\r\nsqueue is busy, will try again\r\n```\r\nAnd NiPype does _not_exit\r\n\r\n### Platform details:\r\n```\r\n(NiPypeUpdate) [zhoud4@felix ETPB]$ python -c \"import nipype; from pprint import pprint; pprint(nipype.get_info())\"\r\n{'commit_hash': 'ec7457c23',\r\n 'commit_source': 'installation',\r\n 'networkx_version': '2.2',\r\n 'nibabel_version': '2.3.1',\r\n 'nipype_version': '1.1.3',\r\n 'numpy_version': '1.15.3',\r\n 'pkg_path': '/data/zhoud4/python/envs/NiPypeUpdate/lib/python3.5/site-packages/nipype',\r\n 'scipy_version': '1.1.0',\r\n 'sys_executable': '/data/zhoud4/python/envs/NiPypeUpdate/bin/python',\r\n 'sys_platform': 'linux',\r\n 'sys_version': '3.5.4 | packaged by conda-forge | (default, Aug 10 2017, '\r\n                '01:38:41) \\n'\r\n                '[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]',\r\n 'traits_version': '4.6.0'}\r\n(NiPypeUpdate) [zhoud4@felix ETPB]$\r\n(NiPypeUpdate) [zhoud4@biowulf ETPB]$ sinfo -V\r\nslurm 17.02.9\r\n(NiPypeUpdate) [zhoud4@biowulf ETPB]$ \r\n```\r\n\n", "before_files": [{"content": "'''\nCreated on Aug 2, 2013\n\n@author: chadcumba\n\nParallel workflow execution with SLURM\n'''\nfrom __future__ import (print_function, division, unicode_literals,\n                        absolute_import)\nfrom builtins import open\n\nimport os\nimport re\nfrom time import sleep\n\nfrom ... import logging\nfrom ...interfaces.base import CommandLine\nfrom .base import SGELikeBatchManagerBase, logger\n\niflogger = logging.getLogger('nipype.interface')\n\n\nclass SLURMPlugin(SGELikeBatchManagerBase):\n    '''\n    Execute using SLURM\n\n    The plugin_args input to run can be used to control the SLURM execution.\n    Currently supported options are:\n\n    - template : template to use for batch job submission\n\n    - sbatch_args: arguments to pass prepend to the sbatch call\n\n\n    '''\n\n    def __init__(self, **kwargs):\n\n        template = \"#!/bin/bash\"\n\n        self._retry_timeout = 2\n        self._max_tries = 2\n        self._template = template\n        self._sbatch_args = None\n        self._jobid_re = \"Submitted batch job ([0-9]*)\"\n\n        if 'plugin_args' in kwargs and kwargs['plugin_args']:\n            if 'retry_timeout' in kwargs['plugin_args']:\n                self._retry_timeout = kwargs['plugin_args']['retry_timeout']\n            if 'max_tries' in kwargs['plugin_args']:\n                self._max_tries = kwargs['plugin_args']['max_tries']\n            if 'jobid_re' in kwargs['plugin_args']:\n                self._jobid_re = kwargs['plugin_args']['jobid_re']\n            if 'template' in kwargs['plugin_args']:\n                self._template = kwargs['plugin_args']['template']\n                if os.path.isfile(self._template):\n                    with open(self._template) as f:\n                        self._template = f.read()\n            if 'sbatch_args' in kwargs['plugin_args']:\n                self._sbatch_args = kwargs['plugin_args']['sbatch_args']\n        self._pending = {}\n        super(SLURMPlugin, self).__init__(self._template, **kwargs)\n\n    def _is_pending(self, taskid):\n        try:\n            res = CommandLine(\n                'squeue',\n                args=' '.join(['-j', '%s' % taskid]),\n                resource_monitor=False,\n                terminal_output='allatonce').run()\n            return res.runtime.stdout.find(str(taskid)) > -1\n        except RuntimeError as e:\n            if 'Invalid job id' not in str(e):\n                raise(e)\n            return False\n\n    def _submit_batchtask(self, scriptfile, node):\n        \"\"\"\n        This is more or less the _submit_batchtask from sge.py with flipped\n        variable names, different command line switches, and different output\n        formatting/processing\n        \"\"\"\n        cmd = CommandLine(\n            'sbatch',\n            environ=dict(os.environ),\n            resource_monitor=False,\n            terminal_output='allatonce')\n        path = os.path.dirname(scriptfile)\n\n        sbatch_args = ''\n        if self._sbatch_args:\n            sbatch_args = self._sbatch_args\n        if 'sbatch_args' in node.plugin_args:\n            if 'overwrite' in node.plugin_args and\\\n               node.plugin_args['overwrite']:\n                sbatch_args = node.plugin_args['sbatch_args']\n            else:\n                sbatch_args += (\" \" + node.plugin_args['sbatch_args'])\n        if '-o' not in sbatch_args:\n            sbatch_args = '%s -o %s' % (sbatch_args,\n                                        os.path.join(path, 'slurm-%j.out'))\n        if '-e' not in sbatch_args:\n            sbatch_args = '%s -e %s' % (sbatch_args,\n                                        os.path.join(path, 'slurm-%j.out'))\n        if node._hierarchy:\n            jobname = '.'.join((dict(os.environ)['LOGNAME'], node._hierarchy,\n                                node._id))\n        else:\n            jobname = '.'.join((dict(os.environ)['LOGNAME'], node._id))\n        jobnameitems = jobname.split('.')\n        jobnameitems.reverse()\n        jobname = '.'.join(jobnameitems)\n        cmd.inputs.args = '%s -J %s %s' % (sbatch_args, jobname, scriptfile)\n        oldlevel = iflogger.level\n        iflogger.setLevel(logging.getLevelName('CRITICAL'))\n        tries = 0\n        while True:\n            try:\n                result = cmd.run()\n            except Exception as e:\n                if tries < self._max_tries:\n                    tries += 1\n                    # sleep 2 seconds and try again.\n                    sleep(self._retry_timeout)\n                else:\n                    iflogger.setLevel(oldlevel)\n                    raise RuntimeError('\\n'.join(\n                        (('Could not submit sbatch task'\n                          ' for node %s') % node._id, str(e))))\n            else:\n                break\n        logger.debug('Ran command ({0})'.format(cmd.cmdline))\n        iflogger.setLevel(oldlevel)\n        # retrieve taskid\n        lines = [line for line in result.runtime.stdout.split('\\n') if line]\n        taskid = int(re.match(self._jobid_re, lines[-1]).groups()[0])\n        self._pending[taskid] = node.output_dir()\n        logger.debug('submitted sbatch task: %d for node %s' % (taskid,\n                                                                node._id))\n        return taskid\n", "path": "nipype/pipeline/plugins/slurm.py"}], "after_files": [{"content": "'''\nCreated on Aug 2, 2013\n\n@author: chadcumba\n\nParallel workflow execution with SLURM\n'''\nfrom __future__ import (print_function, division, unicode_literals,\n                        absolute_import)\nfrom builtins import open\n\nimport os\nimport re\nfrom time import sleep\n\nfrom ... import logging\nfrom ...interfaces.base import CommandLine\nfrom .base import SGELikeBatchManagerBase, logger\n\niflogger = logging.getLogger('nipype.interface')\n\n\nclass SLURMPlugin(SGELikeBatchManagerBase):\n    '''\n    Execute using SLURM\n\n    The plugin_args input to run can be used to control the SLURM execution.\n    Currently supported options are:\n\n    - template : template to use for batch job submission\n\n    - sbatch_args: arguments to pass prepend to the sbatch call\n\n\n    '''\n\n    def __init__(self, **kwargs):\n\n        template = \"#!/bin/bash\"\n\n        self._retry_timeout = 2\n        self._max_tries = 2\n        self._template = template\n        self._sbatch_args = None\n        self._jobid_re = \"Submitted batch job ([0-9]*)\"\n\n        if 'plugin_args' in kwargs and kwargs['plugin_args']:\n            if 'retry_timeout' in kwargs['plugin_args']:\n                self._retry_timeout = kwargs['plugin_args']['retry_timeout']\n            if 'max_tries' in kwargs['plugin_args']:\n                self._max_tries = kwargs['plugin_args']['max_tries']\n            if 'jobid_re' in kwargs['plugin_args']:\n                self._jobid_re = kwargs['plugin_args']['jobid_re']\n            if 'template' in kwargs['plugin_args']:\n                self._template = kwargs['plugin_args']['template']\n                if os.path.isfile(self._template):\n                    with open(self._template) as f:\n                        self._template = f.read()\n            if 'sbatch_args' in kwargs['plugin_args']:\n                self._sbatch_args = kwargs['plugin_args']['sbatch_args']\n        self._pending = {}\n        super(SLURMPlugin, self).__init__(self._template, **kwargs)\n\n    def _is_pending(self, taskid):\n        try:\n            res = CommandLine(\n                'squeue',\n                args=' '.join(['-j', '%s' % taskid]),\n                resource_monitor=False,\n                terminal_output='allatonce').run()\n            return res.runtime.stdout.find(str(taskid)) > -1\n        except RuntimeError as e:\n            if any(ss in str(e) for ss\n                   in ['Socket timed out', 'not available at the moment']):\n                # do not raise error and allow recheck\n                logger.warning(\n                    \"SLURM timeout encountered while checking job status,\"\n                    \" treating job %d as pending\", taskid\n                )\n                return True\n            if 'Invalid job id' not in str(e):\n                raise(e)\n            return False\n\n    def _submit_batchtask(self, scriptfile, node):\n        \"\"\"\n        This is more or less the _submit_batchtask from sge.py with flipped\n        variable names, different command line switches, and different output\n        formatting/processing\n        \"\"\"\n        cmd = CommandLine(\n            'sbatch',\n            environ=dict(os.environ),\n            resource_monitor=False,\n            terminal_output='allatonce')\n        path = os.path.dirname(scriptfile)\n\n        sbatch_args = ''\n        if self._sbatch_args:\n            sbatch_args = self._sbatch_args\n        if 'sbatch_args' in node.plugin_args:\n            if 'overwrite' in node.plugin_args and\\\n               node.plugin_args['overwrite']:\n                sbatch_args = node.plugin_args['sbatch_args']\n            else:\n                sbatch_args += (\" \" + node.plugin_args['sbatch_args'])\n        if '-o' not in sbatch_args:\n            sbatch_args = '%s -o %s' % (sbatch_args,\n                                        os.path.join(path, 'slurm-%j.out'))\n        if '-e' not in sbatch_args:\n            sbatch_args = '%s -e %s' % (sbatch_args,\n                                        os.path.join(path, 'slurm-%j.out'))\n        if node._hierarchy:\n            jobname = '.'.join((dict(os.environ)['LOGNAME'], node._hierarchy,\n                                node._id))\n        else:\n            jobname = '.'.join((dict(os.environ)['LOGNAME'], node._id))\n        jobnameitems = jobname.split('.')\n        jobnameitems.reverse()\n        jobname = '.'.join(jobnameitems)\n        cmd.inputs.args = '%s -J %s %s' % (sbatch_args, jobname, scriptfile)\n        oldlevel = iflogger.level\n        iflogger.setLevel(logging.getLevelName('CRITICAL'))\n        tries = 0\n        while True:\n            try:\n                result = cmd.run()\n            except Exception as e:\n                if tries < self._max_tries:\n                    tries += 1\n                    # sleep 2 seconds and try again.\n                    sleep(self._retry_timeout)\n                else:\n                    iflogger.setLevel(oldlevel)\n                    raise RuntimeError('\\n'.join(\n                        (('Could not submit sbatch task'\n                          ' for node %s') % node._id, str(e))))\n            else:\n                break\n        logger.debug('Ran command ({0})'.format(cmd.cmdline))\n        iflogger.setLevel(oldlevel)\n        # retrieve taskid\n        lines = [line for line in result.runtime.stdout.split('\\n') if line]\n        taskid = int(re.match(self._jobid_re, lines[-1]).groups()[0])\n        self._pending[taskid] = node.output_dir()\n        logger.debug('submitted sbatch task: %d for node %s' % (taskid,\n                                                                node._id))\n        return taskid\n", "path": "nipype/pipeline/plugins/slurm.py"}]}