We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
HunposTagger error in Python3
Bug report from Tülin Erçelebi Ayyıldız:

> Dear hunpos authors,
> 
> I am currently trying to use hunpos tagger to tag a text file, however I get an error at the stage of hunpos.py run.
> 
> My configuration:
> OS : Windows 7-64bit
> python 3.4.1
> nltk 3.0.1
> 
> All "english.model", "hunpos-tag.exe" and "hunpos-train.exe" are located in "C:/Users" folder. My python code is as follows:
> ---

``` python
import nltk 
from nltk.tag.hunpos import HunposTagger
from nltk.tokenize import word_tokenize
corpus = "so how do i hunpos tag my ntuen ? i can't get the following code to work."

ht = HunposTagger('C:/Users/english.model','C:/Users/hunpos-tag.exe')
x=word_tokenize(corpus)
ht.tag(x)
```

> ---
> 
> When I run this module I get the following error:

```
Traceback (most recent call last):
  File "C:\Users\Tülin\Desktop\hunpos_deneme.py", line 12, in <module>
    ht.tag(x)
  File "C:\Python34\lib\site-packages\nltk\tag\hunpos.py", line 109, in tag
    self._hunpos.stdin.write(token + "\n")
TypeError: can't concat bytes to str
```

> I tried several things, but I could not successfully eliminate the problem and get a correct result.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nltk/tag/hunpos.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # Natural Language Toolkit: Interface to the HunPos POS-tagger
3 #
4 # Copyright (C) 2001-2015 NLTK Project
5 # Author: Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
6 #         David Nemeskey <nemeskeyd@gmail.com> (modifications)
7 #         Attila Zseder <zseder@gmail.com> (modifications)
8 # URL: <http://nltk.org/>
9 # For license information, see LICENSE.TXT
10 
11 """
12 A module for interfacing with the HunPos open-source POS-tagger.
13 """
14 
15 import os
16 from subprocess import Popen, PIPE
17 
18 from nltk.internals import find_binary, find_file
19 from nltk.tag.api import TaggerI
20 from nltk import compat
21 
22 _hunpos_url = 'http://code.google.com/p/hunpos/'
23 
24 _hunpos_charset = 'ISO-8859-1'
25 """The default encoding used by hunpos: ISO-8859-1."""
26 
27 class HunposTagger(TaggerI):
28     """
29     A class for pos tagging with HunPos. The input is the paths to:
30      - a model trained on training data
31      - (optionally) the path to the hunpos-tag binary
32      - (optionally) the encoding of the training data (default: ISO-8859-1)
33 
34     Example:
35 
36         >>> from nltk.tag.hunpos import HunposTagger
37         >>> ht = HunposTagger('en_wsj.model')
38         >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())
39         [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
40         >>> ht.close()
41 
42     This class communicates with the hunpos-tag binary via pipes. When the
43     tagger object is no longer needed, the close() method should be called to
44     free system resources. The class supports the context manager interface; if
45     used in a with statement, the close() method is invoked automatically:
46 
47         >>> with HunposTagger('en_wsj.model') as ht:
48         ...     ht.tag('What is the airspeed of an unladen swallow ?'.split())
49         ...
50         [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
51     """
52 
53     def __init__(self, path_to_model, path_to_bin=None,
54                  encoding=_hunpos_charset, verbose=False):
55         """
56         Starts the hunpos-tag executable and establishes a connection with it.
57 
58         :param path_to_model: The model file.
59         :param path_to_bin: The hunpos-tag binary.
60         :param encoding: The encoding used by the model. Unicode tokens
61             passed to the tag() and tag_sents() methods are converted to
62             this charset when they are sent to hunpos-tag.
63             The default is ISO-8859-1 (Latin-1).
64 
65             This parameter is ignored for str tokens, which are sent as-is.
66             The caller must ensure that tokens are encoded in the right charset.
67         """
68         self._closed = True
69         hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',
70                         '/Applications/bin', '~/bin', '~/Applications/bin']
71         hunpos_paths = list(map(os.path.expanduser, hunpos_paths))
72 
73         self._hunpos_bin = find_binary(
74                 'hunpos-tag', path_to_bin,
75                 env_vars=('HUNPOS_TAGGER',),
76                 searchpath=hunpos_paths,
77                 url=_hunpos_url,
78                 verbose=verbose)
79 
80         self._hunpos_model = find_file(path_to_model,
81                 env_vars=('HUNPOS_TAGGER',), verbose=verbose)
82         self._encoding = encoding
83         self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],
84                              shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
85         self._closed = False
86 
87     def __del__(self):
88         self.close()
89 
90     def close(self):
91         """Closes the pipe to the hunpos executable."""
92         if not self._closed:
93             self._hunpos.communicate()
94             self._closed = True
95 
96     def __enter__(self):
97         return self
98     def __exit__(self, exc_type, exc_value, traceback):
99         self.close()
100 
101     def tag(self, tokens):
102         """Tags a single sentence: a list of words.
103         The tokens should not contain any newline characters.
104         """
105         for token in tokens:
106             assert "\n" not in token, "Tokens should not contain newlines"
107             if isinstance(token, compat.text_type):
108                 token = token.encode(self._encoding)
109             self._hunpos.stdin.write(token + "\n")
110         # We write a final empty line to tell hunpos that the sentence is finished:
111         self._hunpos.stdin.write("\n")
112         self._hunpos.stdin.flush()
113 
114         tagged_tokens = []
115         for token in tokens:
116             tagged = self._hunpos.stdout.readline().strip().split("\t")
117             tag = (tagged[1] if len(tagged) > 1 else None)
118             tagged_tokens.append((token, tag))
119         # We have to read (and dismiss) the final empty line:
120         self._hunpos.stdout.readline()
121 
122         return tagged_tokens
123 
124 # skip doctests if Hunpos tagger is not installed
125 def setup_module(module):
126     from nose import SkipTest
127     try:
128         HunposTagger('en_wsj.model')
129     except LookupError:
130         raise SkipTest("HunposTagger is not available")
131 
132 if __name__ == "__main__":
133     import doctest
134     doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
135
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nltk/tag/hunpos.py b/nltk/tag/hunpos.py
--- a/nltk/tag/hunpos.py
+++ b/nltk/tag/hunpos.py
@@ -3,8 +3,8 @@
 #
 # Copyright (C) 2001-2015 NLTK Project
 # Author: Peter Ljunglöf <peter.ljunglof@heatherleaf.se>
-#         David Nemeskey <nemeskeyd@gmail.com> (modifications)
-#         Attila Zseder <zseder@gmail.com> (modifications)
+#         Dávid Márk Nemeskey <nemeskeyd@gmail.com> (modifications)
+#         Attila Zséder <zseder@gmail.com> (modifications)
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
@@ -71,14 +71,15 @@
         hunpos_paths = list(map(os.path.expanduser, hunpos_paths))
 
         self._hunpos_bin = find_binary(
-                'hunpos-tag', path_to_bin,
-                env_vars=('HUNPOS_TAGGER',),
-                searchpath=hunpos_paths,
-                url=_hunpos_url,
-                verbose=verbose)
-
-        self._hunpos_model = find_file(path_to_model,
-                env_vars=('HUNPOS_TAGGER',), verbose=verbose)
+            'hunpos-tag', path_to_bin,
+            env_vars=('HUNPOS_TAGGER',),
+            searchpath=hunpos_paths,
+            url=_hunpos_url,
+            verbose=verbose
+        )
+
+        self._hunpos_model = find_file(
+            path_to_model, env_vars=('HUNPOS_TAGGER',), verbose=verbose)
         self._encoding = encoding
         self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],
                              shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
@@ -106,14 +107,14 @@
             assert "\n" not in token, "Tokens should not contain newlines"
             if isinstance(token, compat.text_type):
                 token = token.encode(self._encoding)
-            self._hunpos.stdin.write(token + "\n")
+            self._hunpos.stdin.write(token + b"\n")
         # We write a final empty line to tell hunpos that the sentence is finished:
-        self._hunpos.stdin.write("\n")
+        self._hunpos.stdin.write(b"\n")
         self._hunpos.stdin.flush()
 
         tagged_tokens = []
         for token in tokens:
-            tagged = self._hunpos.stdout.readline().strip().split("\t")
+            tagged = self._hunpos.stdout.readline().strip().split(b"\t")
             tag = (tagged[1] if len(tagged) > 1 else None)
             tagged_tokens.append((token, tag))
         # We have to read (and dismiss) the final empty line:

{"golden_diff": "diff --git a/nltk/tag/hunpos.py b/nltk/tag/hunpos.py\n--- a/nltk/tag/hunpos.py\n+++ b/nltk/tag/hunpos.py\n@@ -3,8 +3,8 @@\n #\n # Copyright (C) 2001-2015 NLTK Project\n # Author: Peter Ljungl\u00f6f <peter.ljunglof@heatherleaf.se>\n-#         David Nemeskey <nemeskeyd@gmail.com> (modifications)\n-#         Attila Zseder <zseder@gmail.com> (modifications)\n+#         D\u00e1vid M\u00e1rk Nemeskey <nemeskeyd@gmail.com> (modifications)\n+#         Attila Zs\u00e9der <zseder@gmail.com> (modifications)\n # URL: <http://nltk.org/>\n # For license information, see LICENSE.TXT\n \n@@ -71,14 +71,15 @@\n         hunpos_paths = list(map(os.path.expanduser, hunpos_paths))\n \n         self._hunpos_bin = find_binary(\n-                'hunpos-tag', path_to_bin,\n-                env_vars=('HUNPOS_TAGGER',),\n-                searchpath=hunpos_paths,\n-                url=_hunpos_url,\n-                verbose=verbose)\n-\n-        self._hunpos_model = find_file(path_to_model,\n-                env_vars=('HUNPOS_TAGGER',), verbose=verbose)\n+            'hunpos-tag', path_to_bin,\n+            env_vars=('HUNPOS_TAGGER',),\n+            searchpath=hunpos_paths,\n+            url=_hunpos_url,\n+            verbose=verbose\n+        )\n+\n+        self._hunpos_model = find_file(\n+            path_to_model, env_vars=('HUNPOS_TAGGER',), verbose=verbose)\n         self._encoding = encoding\n         self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],\n                              shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)\n@@ -106,14 +107,14 @@\n             assert \"\\n\" not in token, \"Tokens should not contain newlines\"\n             if isinstance(token, compat.text_type):\n                 token = token.encode(self._encoding)\n-            self._hunpos.stdin.write(token + \"\\n\")\n+            self._hunpos.stdin.write(token + b\"\\n\")\n         # We write a final empty line to tell hunpos that the sentence is finished:\n-        self._hunpos.stdin.write(\"\\n\")\n+        self._hunpos.stdin.write(b\"\\n\")\n         self._hunpos.stdin.flush()\n \n         tagged_tokens = []\n         for token in tokens:\n-            tagged = self._hunpos.stdout.readline().strip().split(\"\\t\")\n+            tagged = self._hunpos.stdout.readline().strip().split(b\"\\t\")\n             tag = (tagged[1] if len(tagged) > 1 else None)\n             tagged_tokens.append((token, tag))\n         # We have to read (and dismiss) the final empty line:\n", "issue": "HunposTagger error in Python3\nBug report from T\u00fclin Er\u00e7elebi Ayy\u0131ld\u0131z:\n\n> Dear hunpos authors,\n> \n> I am currently trying to use hunpos tagger to tag a text file, however I get an error at the stage of hunpos.py run.\n> \n> My configuration:\n> OS : Windows 7-64bit\n> python 3.4.1\n> nltk 3.0.1\n> \n> All \"english.model\", \"hunpos-tag.exe\" and \"hunpos-train.exe\" are located in \"C:/Users\" folder. My python code is as follows:\n> ---\n\n``` python\nimport nltk \nfrom nltk.tag.hunpos import HunposTagger\nfrom nltk.tokenize import word_tokenize\ncorpus = \"so how do i hunpos tag my ntuen ? i can't get the following code to work.\"\n\nht = HunposTagger('C:/Users/english.model','C:/Users/hunpos-tag.exe')\nx=word_tokenize(corpus)\nht.tag(x)\n```\n\n> ---\n> \n> When I run this module I get the following error:\n\n```\nTraceback (most recent call last):\n  File \"C:\\Users\\T\u00fclin\\Desktop\\hunpos_deneme.py\", line 12, in <module>\n    ht.tag(x)\n  File \"C:\\Python34\\lib\\site-packages\\nltk\\tag\\hunpos.py\", line 109, in tag\n    self._hunpos.stdin.write(token + \"\\n\")\nTypeError: can't concat bytes to str\n```\n\n> I tried several things, but I could not successfully eliminate the problem and get a correct result.\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# Natural Language Toolkit: Interface to the HunPos POS-tagger\n#\n# Copyright (C) 2001-2015 NLTK Project\n# Author: Peter Ljungl\u00f6f <peter.ljunglof@heatherleaf.se>\n#         David Nemeskey <nemeskeyd@gmail.com> (modifications)\n#         Attila Zseder <zseder@gmail.com> (modifications)\n# URL: <http://nltk.org/>\n# For license information, see LICENSE.TXT\n\n\"\"\"\nA module for interfacing with the HunPos open-source POS-tagger.\n\"\"\"\n\nimport os\nfrom subprocess import Popen, PIPE\n\nfrom nltk.internals import find_binary, find_file\nfrom nltk.tag.api import TaggerI\nfrom nltk import compat\n\n_hunpos_url = 'http://code.google.com/p/hunpos/'\n\n_hunpos_charset = 'ISO-8859-1'\n\"\"\"The default encoding used by hunpos: ISO-8859-1.\"\"\"\n\nclass HunposTagger(TaggerI):\n    \"\"\"\n    A class for pos tagging with HunPos. The input is the paths to:\n     - a model trained on training data\n     - (optionally) the path to the hunpos-tag binary\n     - (optionally) the encoding of the training data (default: ISO-8859-1)\n\n    Example:\n\n        >>> from nltk.tag.hunpos import HunposTagger\n        >>> ht = HunposTagger('en_wsj.model')\n        >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())\n        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]\n        >>> ht.close()\n\n    This class communicates with the hunpos-tag binary via pipes. When the\n    tagger object is no longer needed, the close() method should be called to\n    free system resources. The class supports the context manager interface; if\n    used in a with statement, the close() method is invoked automatically:\n\n        >>> with HunposTagger('en_wsj.model') as ht:\n        ...     ht.tag('What is the airspeed of an unladen swallow ?'.split())\n        ...\n        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]\n    \"\"\"\n\n    def __init__(self, path_to_model, path_to_bin=None,\n                 encoding=_hunpos_charset, verbose=False):\n        \"\"\"\n        Starts the hunpos-tag executable and establishes a connection with it.\n\n        :param path_to_model: The model file.\n        :param path_to_bin: The hunpos-tag binary.\n        :param encoding: The encoding used by the model. Unicode tokens\n            passed to the tag() and tag_sents() methods are converted to\n            this charset when they are sent to hunpos-tag.\n            The default is ISO-8859-1 (Latin-1).\n\n            This parameter is ignored for str tokens, which are sent as-is.\n            The caller must ensure that tokens are encoded in the right charset.\n        \"\"\"\n        self._closed = True\n        hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',\n                        '/Applications/bin', '~/bin', '~/Applications/bin']\n        hunpos_paths = list(map(os.path.expanduser, hunpos_paths))\n\n        self._hunpos_bin = find_binary(\n                'hunpos-tag', path_to_bin,\n                env_vars=('HUNPOS_TAGGER',),\n                searchpath=hunpos_paths,\n                url=_hunpos_url,\n                verbose=verbose)\n\n        self._hunpos_model = find_file(path_to_model,\n                env_vars=('HUNPOS_TAGGER',), verbose=verbose)\n        self._encoding = encoding\n        self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],\n                             shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)\n        self._closed = False\n\n    def __del__(self):\n        self.close()\n\n    def close(self):\n        \"\"\"Closes the pipe to the hunpos executable.\"\"\"\n        if not self._closed:\n            self._hunpos.communicate()\n            self._closed = True\n\n    def __enter__(self):\n        return self\n    def __exit__(self, exc_type, exc_value, traceback):\n        self.close()\n\n    def tag(self, tokens):\n        \"\"\"Tags a single sentence: a list of words.\n        The tokens should not contain any newline characters.\n        \"\"\"\n        for token in tokens:\n            assert \"\\n\" not in token, \"Tokens should not contain newlines\"\n            if isinstance(token, compat.text_type):\n                token = token.encode(self._encoding)\n            self._hunpos.stdin.write(token + \"\\n\")\n        # We write a final empty line to tell hunpos that the sentence is finished:\n        self._hunpos.stdin.write(\"\\n\")\n        self._hunpos.stdin.flush()\n\n        tagged_tokens = []\n        for token in tokens:\n            tagged = self._hunpos.stdout.readline().strip().split(\"\\t\")\n            tag = (tagged[1] if len(tagged) > 1 else None)\n            tagged_tokens.append((token, tag))\n        # We have to read (and dismiss) the final empty line:\n        self._hunpos.stdout.readline()\n\n        return tagged_tokens\n\n# skip doctests if Hunpos tagger is not installed\ndef setup_module(module):\n    from nose import SkipTest\n    try:\n        HunposTagger('en_wsj.model')\n    except LookupError:\n        raise SkipTest(\"HunposTagger is not available\")\n\nif __name__ == \"__main__\":\n    import doctest\n    doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)\n", "path": "nltk/tag/hunpos.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# Natural Language Toolkit: Interface to the HunPos POS-tagger\n#\n# Copyright (C) 2001-2015 NLTK Project\n# Author: Peter Ljungl\u00f6f <peter.ljunglof@heatherleaf.se>\n#         D\u00e1vid M\u00e1rk Nemeskey <nemeskeyd@gmail.com> (modifications)\n#         Attila Zs\u00e9der <zseder@gmail.com> (modifications)\n# URL: <http://nltk.org/>\n# For license information, see LICENSE.TXT\n\n\"\"\"\nA module for interfacing with the HunPos open-source POS-tagger.\n\"\"\"\n\nimport os\nfrom subprocess import Popen, PIPE\n\nfrom nltk.internals import find_binary, find_file\nfrom nltk.tag.api import TaggerI\nfrom nltk import compat\n\n_hunpos_url = 'http://code.google.com/p/hunpos/'\n\n_hunpos_charset = 'ISO-8859-1'\n\"\"\"The default encoding used by hunpos: ISO-8859-1.\"\"\"\n\nclass HunposTagger(TaggerI):\n    \"\"\"\n    A class for pos tagging with HunPos. The input is the paths to:\n     - a model trained on training data\n     - (optionally) the path to the hunpos-tag binary\n     - (optionally) the encoding of the training data (default: ISO-8859-1)\n\n    Example:\n\n        >>> from nltk.tag.hunpos import HunposTagger\n        >>> ht = HunposTagger('en_wsj.model')\n        >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())\n        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]\n        >>> ht.close()\n\n    This class communicates with the hunpos-tag binary via pipes. When the\n    tagger object is no longer needed, the close() method should be called to\n    free system resources. The class supports the context manager interface; if\n    used in a with statement, the close() method is invoked automatically:\n\n        >>> with HunposTagger('en_wsj.model') as ht:\n        ...     ht.tag('What is the airspeed of an unladen swallow ?'.split())\n        ...\n        [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]\n    \"\"\"\n\n    def __init__(self, path_to_model, path_to_bin=None,\n                 encoding=_hunpos_charset, verbose=False):\n        \"\"\"\n        Starts the hunpos-tag executable and establishes a connection with it.\n\n        :param path_to_model: The model file.\n        :param path_to_bin: The hunpos-tag binary.\n        :param encoding: The encoding used by the model. Unicode tokens\n            passed to the tag() and tag_sents() methods are converted to\n            this charset when they are sent to hunpos-tag.\n            The default is ISO-8859-1 (Latin-1).\n\n            This parameter is ignored for str tokens, which are sent as-is.\n            The caller must ensure that tokens are encoded in the right charset.\n        \"\"\"\n        self._closed = True\n        hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',\n                        '/Applications/bin', '~/bin', '~/Applications/bin']\n        hunpos_paths = list(map(os.path.expanduser, hunpos_paths))\n\n        self._hunpos_bin = find_binary(\n            'hunpos-tag', path_to_bin,\n            env_vars=('HUNPOS_TAGGER',),\n            searchpath=hunpos_paths,\n            url=_hunpos_url,\n            verbose=verbose\n        )\n\n        self._hunpos_model = find_file(\n            path_to_model, env_vars=('HUNPOS_TAGGER',), verbose=verbose)\n        self._encoding = encoding\n        self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],\n                             shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)\n        self._closed = False\n\n    def __del__(self):\n        self.close()\n\n    def close(self):\n        \"\"\"Closes the pipe to the hunpos executable.\"\"\"\n        if not self._closed:\n            self._hunpos.communicate()\n            self._closed = True\n\n    def __enter__(self):\n        return self\n    def __exit__(self, exc_type, exc_value, traceback):\n        self.close()\n\n    def tag(self, tokens):\n        \"\"\"Tags a single sentence: a list of words.\n        The tokens should not contain any newline characters.\n        \"\"\"\n        for token in tokens:\n            assert \"\\n\" not in token, \"Tokens should not contain newlines\"\n            if isinstance(token, compat.text_type):\n                token = token.encode(self._encoding)\n            self._hunpos.stdin.write(token + b\"\\n\")\n        # We write a final empty line to tell hunpos that the sentence is finished:\n        self._hunpos.stdin.write(b\"\\n\")\n        self._hunpos.stdin.flush()\n\n        tagged_tokens = []\n        for token in tokens:\n            tagged = self._hunpos.stdout.readline().strip().split(b\"\\t\")\n            tag = (tagged[1] if len(tagged) > 1 else None)\n            tagged_tokens.append((token, tag))\n        # We have to read (and dismiss) the final empty line:\n        self._hunpos.stdout.readline()\n\n        return tagged_tokens\n\n# skip doctests if Hunpos tagger is not installed\ndef setup_module(module):\n    from nose import SkipTest\n    try:\n        HunposTagger('en_wsj.model')\n    except LookupError:\n        raise SkipTest(\"HunposTagger is not available\")\n\nif __name__ == \"__main__\":\n    import doctest\n    doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)\n", "path": "nltk/tag/hunpos.py"}]}