We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
grid engine provider hangs when qstat output is large
**Describe the bug**

From Quentin Le Boulc'h;

```
I still have the issue with the qstat command from Parsl which is hitting the 60s timeout. When I have 1000 jobs running every qstat fails, I can see many processes accumulating and Parsl get completely stucked.
I checked I can reproduce the issue by myself running the subprocess command you are using. With several hundreds of jobs running the proc.wait() is waiting forever. However a manual qstat returns the output in less than a second. And after killing the proc.wait()  I can check that the stdout is actually here so the command has succeeded.
I can also reproduce the issue with a different command than `qstat`, like a simple script returning several hundreds of lines.
Could you check if there is some issue with your subprocess command?
See https://docs.python.org/3/library/subprocess.html#popen-objects for instance: “This will deadlock when using stdout=PIPE or stderr=PIPE and the child process generates enough output to a pipe such that it blocks waiting for the OS pipe buffer to accept more data. Use Popen.communicate() when using pipes to avoid that.”
```


**Expected behavior**
This should not hang

**Environment**
in2p3, parsl desc branch

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `parsl/channels/local/local.py`
Content:
```
1 import copy
2 import logging
3 import os
4 import shutil
5 import subprocess
6 
7 from parsl.channels.base import Channel
8 from parsl.channels.errors import FileCopyException
9 from parsl.utils import RepresentationMixin
10 
11 logger = logging.getLogger(__name__)
12 
13 
14 class LocalChannel(Channel, RepresentationMixin):
15     ''' This is not even really a channel, since opening a local shell is not heavy
16     and done so infrequently that they do not need a persistent channel
17     '''
18 
19     def __init__(self, userhome=".", envs={}, script_dir=None):
20         ''' Initialize the local channel. script_dir is required by set to a default.
21 
22         KwArgs:
23             - userhome (string): (default='.') This is provided as a way to override and set a specific userhome
24             - envs (dict) : A dictionary of env variables to be set when launching the shell
25             - script_dir (string): Directory to place scripts
26         '''
27         self.userhome = os.path.abspath(userhome)
28         self.hostname = "localhost"
29         self.envs = envs
30         local_env = os.environ.copy()
31         self._envs = copy.deepcopy(local_env)
32         self._envs.update(envs)
33         self.script_dir = script_dir
34 
35     def execute_wait(self, cmd, walltime=None, envs={}):
36         ''' Synchronously execute a commandline string on the shell.
37 
38         Args:
39             - cmd (string) : Commandline string to execute
40             - walltime (int) : walltime in seconds, this is not really used now.
41 
42         Kwargs:
43             - envs (dict) : Dictionary of env variables. This will be used
44               to override the envs set at channel initialization.
45 
46         Returns:
47             - retcode : Return code from the execution, -1 on fail
48             - stdout  : stdout string
49             - stderr  : stderr string
50 
51         Raises:
52         None.
53         '''
54         retcode = -1
55         stdout = None
56         stderr = None
57 
58         current_env = copy.deepcopy(self._envs)
59         current_env.update(envs)
60 
61         try:
62             proc = subprocess.Popen(
63                 cmd,
64                 stdout=subprocess.PIPE,
65                 stderr=subprocess.PIPE,
66                 cwd=self.userhome,
67                 env=current_env,
68                 shell=True,
69                 preexec_fn=os.setpgrp
70             )
71             proc.wait(timeout=walltime)
72             stdout = proc.stdout.read()
73             stderr = proc.stderr.read()
74             retcode = proc.returncode
75 
76         except Exception as e:
77             logger.warning("Execution of command '{}' failed due to \n{}".format(cmd, e))
78             raise
79 
80         return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))
81 
82     def push_file(self, source, dest_dir):
83         ''' If the source files dirpath is the same as dest_dir, a copy
84         is not necessary, and nothing is done. Else a copy is made.
85 
86         Args:
87             - source (string) : Path to the source file
88             - dest_dir (string) : Path to the directory to which the files is to be copied
89 
90         Returns:
91             - destination_path (String) : Absolute path of the destination file
92 
93         Raises:
94             - FileCopyException : If file copy failed.
95         '''
96 
97         local_dest = os.path.join(dest_dir, os.path.basename(source))
98 
99         # Only attempt to copy if the target dir and source dir are different
100         if os.path.dirname(source) != dest_dir:
101             try:
102                 shutil.copyfile(source, local_dest)
103                 os.chmod(local_dest, 0o777)
104 
105             except OSError as e:
106                 raise FileCopyException(e, self.hostname)
107 
108         else:
109             os.chmod(local_dest, 0o777)
110 
111         return local_dest
112 
113     def pull_file(self, remote_source, local_dir):
114         return self.push_file(remote_source, local_dir)
115 
116     def close(self):
117         ''' There's nothing to close here, and this really doesn't do anything
118 
119         Returns:
120              - False, because it really did not "close" this channel.
121         '''
122         return False
123 
124     def isdir(self, path):
125         """Return true if the path refers to an existing directory.
126 
127         Parameters
128         ----------
129         path : str
130             Path of directory to check.
131         """
132 
133         return os.path.isdir(path)
134 
135     def makedirs(self, path, mode=511, exist_ok=False):
136         """Create a directory.
137 
138         If intermediate directories do not exist, they will be created.
139 
140         Parameters
141         ----------
142         path : str
143             Path of directory to create.
144         mode : int
145             Permissions (posix-style) for the newly-created directory.
146         exist_ok : bool
147             If False, raise an OSError if the target directory already exists.
148         """
149 
150         return os.makedirs(path, mode, exist_ok)
151 
152     def abspath(self, path):
153         """Return the absolute path.
154 
155         Parameters
156         ----------
157         path : str
158             Path for which the absolute path will be returned.
159         """
160         return os.path.abspath(path)
161 
162     @property
163     def script_dir(self):
164         return self._script_dir
165 
166     @script_dir.setter
167     def script_dir(self, value):
168         if value is not None:
169             value = self.abspath(value)
170         self._script_dir = value
171
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/parsl/channels/local/local.py b/parsl/channels/local/local.py
--- a/parsl/channels/local/local.py
+++ b/parsl/channels/local/local.py
@@ -68,9 +68,7 @@
                 shell=True,
                 preexec_fn=os.setpgrp
             )
-            proc.wait(timeout=walltime)
-            stdout = proc.stdout.read()
-            stderr = proc.stderr.read()
+            (stdout, stderr) = proc.communicate(timeout=walltime)
             retcode = proc.returncode
 
         except Exception as e:

{"golden_diff": "diff --git a/parsl/channels/local/local.py b/parsl/channels/local/local.py\n--- a/parsl/channels/local/local.py\n+++ b/parsl/channels/local/local.py\n@@ -68,9 +68,7 @@\n                 shell=True,\n                 preexec_fn=os.setpgrp\n             )\n-            proc.wait(timeout=walltime)\n-            stdout = proc.stdout.read()\n-            stderr = proc.stderr.read()\n+            (stdout, stderr) = proc.communicate(timeout=walltime)\n             retcode = proc.returncode\n \n         except Exception as e:\n", "issue": "grid engine provider hangs when qstat output is large\n**Describe the bug**\r\n\r\nFrom Quentin Le Boulc'h;\r\n\r\n```\r\nI still have the issue with the qstat command from Parsl which is hitting the 60s timeout. When I have 1000 jobs running every qstat fails, I can see many processes accumulating and Parsl get completely stucked.\r\nI checked I can reproduce the issue by myself running the subprocess command you are using. With several hundreds of jobs running the proc.wait() is waiting forever. However a manual qstat returns the output in less than a second. And after killing the proc.wait()  I can check that the stdout is actually here so the command has succeeded.\r\nI can also reproduce the issue with a different command than `qstat`, like a simple script returning several hundreds of lines.\r\nCould you check if there is some issue with your subprocess command?\r\nSee https://docs.python.org/3/library/subprocess.html#popen-objects for instance: \u201cThis will deadlock when using stdout=PIPE or stderr=PIPE and the child process generates enough output to a pipe such that it blocks waiting for the OS pipe buffer to accept more data. Use Popen.communicate() when using pipes to avoid that.\u201d\r\n```\r\n\r\n\r\n**Expected behavior**\r\nThis should not hang\r\n\r\n**Environment**\r\nin2p3, parsl desc branch\n", "before_files": [{"content": "import copy\nimport logging\nimport os\nimport shutil\nimport subprocess\n\nfrom parsl.channels.base import Channel\nfrom parsl.channels.errors import FileCopyException\nfrom parsl.utils import RepresentationMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass LocalChannel(Channel, RepresentationMixin):\n    ''' This is not even really a channel, since opening a local shell is not heavy\n    and done so infrequently that they do not need a persistent channel\n    '''\n\n    def __init__(self, userhome=\".\", envs={}, script_dir=None):\n        ''' Initialize the local channel. script_dir is required by set to a default.\n\n        KwArgs:\n            - userhome (string): (default='.') This is provided as a way to override and set a specific userhome\n            - envs (dict) : A dictionary of env variables to be set when launching the shell\n            - script_dir (string): Directory to place scripts\n        '''\n        self.userhome = os.path.abspath(userhome)\n        self.hostname = \"localhost\"\n        self.envs = envs\n        local_env = os.environ.copy()\n        self._envs = copy.deepcopy(local_env)\n        self._envs.update(envs)\n        self.script_dir = script_dir\n\n    def execute_wait(self, cmd, walltime=None, envs={}):\n        ''' Synchronously execute a commandline string on the shell.\n\n        Args:\n            - cmd (string) : Commandline string to execute\n            - walltime (int) : walltime in seconds, this is not really used now.\n\n        Kwargs:\n            - envs (dict) : Dictionary of env variables. This will be used\n              to override the envs set at channel initialization.\n\n        Returns:\n            - retcode : Return code from the execution, -1 on fail\n            - stdout  : stdout string\n            - stderr  : stderr string\n\n        Raises:\n        None.\n        '''\n        retcode = -1\n        stdout = None\n        stderr = None\n\n        current_env = copy.deepcopy(self._envs)\n        current_env.update(envs)\n\n        try:\n            proc = subprocess.Popen(\n                cmd,\n                stdout=subprocess.PIPE,\n                stderr=subprocess.PIPE,\n                cwd=self.userhome,\n                env=current_env,\n                shell=True,\n                preexec_fn=os.setpgrp\n            )\n            proc.wait(timeout=walltime)\n            stdout = proc.stdout.read()\n            stderr = proc.stderr.read()\n            retcode = proc.returncode\n\n        except Exception as e:\n            logger.warning(\"Execution of command '{}' failed due to \\n{}\".format(cmd, e))\n            raise\n\n        return (retcode, stdout.decode(\"utf-8\"), stderr.decode(\"utf-8\"))\n\n    def push_file(self, source, dest_dir):\n        ''' If the source files dirpath is the same as dest_dir, a copy\n        is not necessary, and nothing is done. Else a copy is made.\n\n        Args:\n            - source (string) : Path to the source file\n            - dest_dir (string) : Path to the directory to which the files is to be copied\n\n        Returns:\n            - destination_path (String) : Absolute path of the destination file\n\n        Raises:\n            - FileCopyException : If file copy failed.\n        '''\n\n        local_dest = os.path.join(dest_dir, os.path.basename(source))\n\n        # Only attempt to copy if the target dir and source dir are different\n        if os.path.dirname(source) != dest_dir:\n            try:\n                shutil.copyfile(source, local_dest)\n                os.chmod(local_dest, 0o777)\n\n            except OSError as e:\n                raise FileCopyException(e, self.hostname)\n\n        else:\n            os.chmod(local_dest, 0o777)\n\n        return local_dest\n\n    def pull_file(self, remote_source, local_dir):\n        return self.push_file(remote_source, local_dir)\n\n    def close(self):\n        ''' There's nothing to close here, and this really doesn't do anything\n\n        Returns:\n             - False, because it really did not \"close\" this channel.\n        '''\n        return False\n\n    def isdir(self, path):\n        \"\"\"Return true if the path refers to an existing directory.\n\n        Parameters\n        ----------\n        path : str\n            Path of directory to check.\n        \"\"\"\n\n        return os.path.isdir(path)\n\n    def makedirs(self, path, mode=511, exist_ok=False):\n        \"\"\"Create a directory.\n\n        If intermediate directories do not exist, they will be created.\n\n        Parameters\n        ----------\n        path : str\n            Path of directory to create.\n        mode : int\n            Permissions (posix-style) for the newly-created directory.\n        exist_ok : bool\n            If False, raise an OSError if the target directory already exists.\n        \"\"\"\n\n        return os.makedirs(path, mode, exist_ok)\n\n    def abspath(self, path):\n        \"\"\"Return the absolute path.\n\n        Parameters\n        ----------\n        path : str\n            Path for which the absolute path will be returned.\n        \"\"\"\n        return os.path.abspath(path)\n\n    @property\n    def script_dir(self):\n        return self._script_dir\n\n    @script_dir.setter\n    def script_dir(self, value):\n        if value is not None:\n            value = self.abspath(value)\n        self._script_dir = value\n", "path": "parsl/channels/local/local.py"}], "after_files": [{"content": "import copy\nimport logging\nimport os\nimport shutil\nimport subprocess\n\nfrom parsl.channels.base import Channel\nfrom parsl.channels.errors import FileCopyException\nfrom parsl.utils import RepresentationMixin\n\nlogger = logging.getLogger(__name__)\n\n\nclass LocalChannel(Channel, RepresentationMixin):\n    ''' This is not even really a channel, since opening a local shell is not heavy\n    and done so infrequently that they do not need a persistent channel\n    '''\n\n    def __init__(self, userhome=\".\", envs={}, script_dir=None):\n        ''' Initialize the local channel. script_dir is required by set to a default.\n\n        KwArgs:\n            - userhome (string): (default='.') This is provided as a way to override and set a specific userhome\n            - envs (dict) : A dictionary of env variables to be set when launching the shell\n            - script_dir (string): Directory to place scripts\n        '''\n        self.userhome = os.path.abspath(userhome)\n        self.hostname = \"localhost\"\n        self.envs = envs\n        local_env = os.environ.copy()\n        self._envs = copy.deepcopy(local_env)\n        self._envs.update(envs)\n        self.script_dir = script_dir\n\n    def execute_wait(self, cmd, walltime=None, envs={}):\n        ''' Synchronously execute a commandline string on the shell.\n\n        Args:\n            - cmd (string) : Commandline string to execute\n            - walltime (int) : walltime in seconds, this is not really used now.\n\n        Kwargs:\n            - envs (dict) : Dictionary of env variables. This will be used\n              to override the envs set at channel initialization.\n\n        Returns:\n            - retcode : Return code from the execution, -1 on fail\n            - stdout  : stdout string\n            - stderr  : stderr string\n\n        Raises:\n        None.\n        '''\n        retcode = -1\n        stdout = None\n        stderr = None\n\n        current_env = copy.deepcopy(self._envs)\n        current_env.update(envs)\n\n        try:\n            proc = subprocess.Popen(\n                cmd,\n                stdout=subprocess.PIPE,\n                stderr=subprocess.PIPE,\n                cwd=self.userhome,\n                env=current_env,\n                shell=True,\n                preexec_fn=os.setpgrp\n            )\n            (stdout, stderr) = proc.communicate(timeout=walltime)\n            retcode = proc.returncode\n\n        except Exception as e:\n            logger.warning(\"Execution of command '{}' failed due to \\n{}\".format(cmd, e))\n            raise\n\n        return (retcode, stdout.decode(\"utf-8\"), stderr.decode(\"utf-8\"))\n\n    def push_file(self, source, dest_dir):\n        ''' If the source files dirpath is the same as dest_dir, a copy\n        is not necessary, and nothing is done. Else a copy is made.\n\n        Args:\n            - source (string) : Path to the source file\n            - dest_dir (string) : Path to the directory to which the files is to be copied\n\n        Returns:\n            - destination_path (String) : Absolute path of the destination file\n\n        Raises:\n            - FileCopyException : If file copy failed.\n        '''\n\n        local_dest = os.path.join(dest_dir, os.path.basename(source))\n\n        # Only attempt to copy if the target dir and source dir are different\n        if os.path.dirname(source) != dest_dir:\n            try:\n                shutil.copyfile(source, local_dest)\n                os.chmod(local_dest, 0o777)\n\n            except OSError as e:\n                raise FileCopyException(e, self.hostname)\n\n        else:\n            os.chmod(local_dest, 0o777)\n\n        return local_dest\n\n    def pull_file(self, remote_source, local_dir):\n        return self.push_file(remote_source, local_dir)\n\n    def close(self):\n        ''' There's nothing to close here, and this really doesn't do anything\n\n        Returns:\n             - False, because it really did not \"close\" this channel.\n        '''\n        return False\n\n    def isdir(self, path):\n        \"\"\"Return true if the path refers to an existing directory.\n\n        Parameters\n        ----------\n        path : str\n            Path of directory to check.\n        \"\"\"\n\n        return os.path.isdir(path)\n\n    def makedirs(self, path, mode=511, exist_ok=False):\n        \"\"\"Create a directory.\n\n        If intermediate directories do not exist, they will be created.\n\n        Parameters\n        ----------\n        path : str\n            Path of directory to create.\n        mode : int\n            Permissions (posix-style) for the newly-created directory.\n        exist_ok : bool\n            If False, raise an OSError if the target directory already exists.\n        \"\"\"\n\n        return os.makedirs(path, mode, exist_ok)\n\n    def abspath(self, path):\n        \"\"\"Return the absolute path.\n\n        Parameters\n        ----------\n        path : str\n            Path for which the absolute path will be returned.\n        \"\"\"\n        return os.path.abspath(path)\n\n    @property\n    def script_dir(self):\n        return self._script_dir\n\n    @script_dir.setter\n    def script_dir(self, value):\n        if value is not None:\n            value = self.abspath(value)\n        self._script_dir = value\n", "path": "parsl/channels/local/local.py"}]}