We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Wrong result for word more'n in Word Tokenizer and PTB Tokenizer
Let sentence: `It's more'n enough`

If I'm not mistaken, the PTB Tokenizer should result something like this:

`["It", "'s", "more", "'n", "enough"]`

But, it's not. It returns:

`["It", "'s", "more", "'", "n", "enough"]`

Since Word Tokenizer trying to implement PTB contraction, the result should be like that, right?

**PS**: Word Tokenizer contraction is `mor'n`, while original PTB contraction is `more'n`. Need clarification.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nltk/tokenize/destructive.py`
Content:
```
1 # Natural Language Toolkit: NLTK's very own tokenizer.
2 #
3 # Copyright (C) 2001-2020 NLTK Project
4 # Author:
5 # URL: <http://nltk.sourceforge.net>
6 # For license information, see LICENSE.TXT
7 
8 
9 import re
10 from nltk.tokenize.api import TokenizerI
11 
12 
13 class MacIntyreContractions:
14     """
15     List of contractions adapted from Robert MacIntyre's tokenizer.
16     """
17 
18     CONTRACTIONS2 = [
19         r"(?i)\b(can)(?#X)(not)\b",
20         r"(?i)\b(d)(?#X)('ye)\b",
21         r"(?i)\b(gim)(?#X)(me)\b",
22         r"(?i)\b(gon)(?#X)(na)\b",
23         r"(?i)\b(got)(?#X)(ta)\b",
24         r"(?i)\b(lem)(?#X)(me)\b",
25         r"(?i)\b(mor)(?#X)('n)\b",
26         r"(?i)\b(wan)(?#X)(na)\s",
27     ]
28     CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
29     CONTRACTIONS4 = [r"(?i)\b(whad)(dd)(ya)\b", r"(?i)\b(wha)(t)(cha)\b"]
30 
31 
32 class NLTKWordTokenizer(TokenizerI):
33     """
34     The NLTK tokenizer that has improved upon the TreebankWordTokenizer.
35 
36     The tokenizer is "destructive" such that the regexes applied will munge the
37     input string to a state beyond re-construction. It is possible to apply
38     `TreebankWordDetokenizer.detokenize` to the tokenized outputs of
39     `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to
40     revert to the original string.
41     """
42 
43     # Starting quotes.
44     STARTING_QUOTES = [
45         (re.compile(u"([«“‘„]|[`]+)", re.U), r" \1 "),
46         (re.compile(r"^\""), r"``"),
47         (re.compile(r"(``)"), r" \1 "),
48         (re.compile(r"([ \(\[{<])(\"|\'{2})"), r"\1 `` "),
49         (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d)(\w)\b", re.U), r"\1 \2"),
50     ]
51 
52     # Ending quotes.
53     ENDING_QUOTES = [
54         (re.compile(u"([»”’])", re.U), r" \1 "),
55         (re.compile(r'"'), " '' "),
56         (re.compile(r"(\S)(\'\')"), r"\1 \2 "),
57         (re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
58         (re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
59     ]
60 
61     # For improvements for starting/closing quotes from TreebankWordTokenizer,
62     # see discussion on https://github.com/nltk/nltk/pull/1437
63     # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on
64     # - chervon quotes u'\xab' and u'\xbb' .
65     # - unicode quotes u'\u2018', u'\u2019', u'\u201c' and u'\u201d'
66     # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608
67     # Also, behavior of splitting on clitics now follows Stanford CoreNLP
68     # - clitics covered (?!re|ve|ll|m|t|s|d)(\w)\b
69 
70     # Punctuation.
71     PUNCTUATION = [
72         (re.compile(r'([^\.])(\.)([\]\)}>"\'' u"»”’ " r"]*)\s*$", re.U), r"\1 \2 \3 "),
73         (re.compile(r"([:,])([^\d])"), r" \1 \2"),
74         (re.compile(r"([:,])$"), r" \1 "),
75         (re.compile(r"\.{2,}", re.U), r" \g<0> "), # See https://github.com/nltk/nltk/pull/2322
76         (re.compile(r"[;@#$%&]"), r" \g<0> "),
77         (
78             re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
79             r"\1 \2\3 ",
80         ),  # Handles the final period.
81         (re.compile(r"[?!]"), r" \g<0> "),
82         (re.compile(r"([^'])' "), r"\1 ' "),
83         (re.compile(r"[*]", re.U), r" \g<0> "), # See https://github.com/nltk/nltk/pull/2322
84     ]
85 
86     # Pads parentheses
87     PARENS_BRACKETS = (re.compile(r"[\]\[\(\)\{\}\<\>]"), r" \g<0> ")
88 
89     # Optionally: Convert parentheses, brackets and converts them to PTB symbols.
90     CONVERT_PARENTHESES = [
91         (re.compile(r"\("), "-LRB-"),
92         (re.compile(r"\)"), "-RRB-"),
93         (re.compile(r"\["), "-LSB-"),
94         (re.compile(r"\]"), "-RSB-"),
95         (re.compile(r"\{"), "-LCB-"),
96         (re.compile(r"\}"), "-RCB-"),
97     ]
98 
99     DOUBLE_DASHES = (re.compile(r"--"), r" -- ")
100 
101     # List of contractions adapted from Robert MacIntyre's tokenizer.
102     _contractions = MacIntyreContractions()
103     CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
104     CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
105 
106     def tokenize(self, text, convert_parentheses=False, return_str=False):
107         for regexp, substitution in self.STARTING_QUOTES:
108             text = regexp.sub(substitution, text)
109 
110         for regexp, substitution in self.PUNCTUATION:
111             text = regexp.sub(substitution, text)
112 
113         # Handles parentheses.
114         regexp, substitution = self.PARENS_BRACKETS
115         text = regexp.sub(substitution, text)
116         # Optionally convert parentheses
117         if convert_parentheses:
118             for regexp, substitution in self.CONVERT_PARENTHESES:
119                 text = regexp.sub(substitution, text)
120 
121         # Handles double dash.
122         regexp, substitution = self.DOUBLE_DASHES
123         text = regexp.sub(substitution, text)
124 
125         # add extra space to make things easier
126         text = " " + text + " "
127 
128         for regexp, substitution in self.ENDING_QUOTES:
129             text = regexp.sub(substitution, text)
130 
131         for regexp in self.CONTRACTIONS2:
132             text = regexp.sub(r" \1 \2 ", text)
133         for regexp in self.CONTRACTIONS3:
134             text = regexp.sub(r" \1 \2 ", text)
135 
136         # We are not using CONTRACTIONS4 since
137         # they are also commented out in the SED scripts
138         # for regexp in self._contractions.CONTRACTIONS4:
139         #     text = regexp.sub(r' \1 \2 \3 ', text)
140 
141         return text if return_str else text.split()
142
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nltk/tokenize/destructive.py b/nltk/tokenize/destructive.py
--- a/nltk/tokenize/destructive.py
+++ b/nltk/tokenize/destructive.py
@@ -22,7 +22,7 @@
         r"(?i)\b(gon)(?#X)(na)\b",
         r"(?i)\b(got)(?#X)(ta)\b",
         r"(?i)\b(lem)(?#X)(me)\b",
-        r"(?i)\b(mor)(?#X)('n)\b",
+        r"(?i)\b(more)(?#X)('n)\b",
         r"(?i)\b(wan)(?#X)(na)\s",
     ]
     CONTRACTIONS3 = [r"(?i) ('t)(?#X)(is)\b", r"(?i) ('t)(?#X)(was)\b"]
@@ -46,7 +46,7 @@
         (re.compile(r"^\""), r"``"),
         (re.compile(r"(``)"), r" \1 "),
         (re.compile(r"([ \(\[{<])(\"|\'{2})"), r"\1 `` "),
-        (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d)(\w)\b", re.U), r"\1 \2"),
+        (re.compile(r"(?i)(\')(?!re|ve|ll|m|t|s|d|n)(\w)\b", re.U), r"\1 \2"),
     ]
 
     # Ending quotes.

{"golden_diff": "diff --git a/nltk/tokenize/destructive.py b/nltk/tokenize/destructive.py\n--- a/nltk/tokenize/destructive.py\n+++ b/nltk/tokenize/destructive.py\n@@ -22,7 +22,7 @@\n         r\"(?i)\\b(gon)(?#X)(na)\\b\",\n         r\"(?i)\\b(got)(?#X)(ta)\\b\",\n         r\"(?i)\\b(lem)(?#X)(me)\\b\",\n-        r\"(?i)\\b(mor)(?#X)('n)\\b\",\n+        r\"(?i)\\b(more)(?#X)('n)\\b\",\n         r\"(?i)\\b(wan)(?#X)(na)\\s\",\n     ]\n     CONTRACTIONS3 = [r\"(?i) ('t)(?#X)(is)\\b\", r\"(?i) ('t)(?#X)(was)\\b\"]\n@@ -46,7 +46,7 @@\n         (re.compile(r\"^\\\"\"), r\"``\"),\n         (re.compile(r\"(``)\"), r\" \\1 \"),\n         (re.compile(r\"([ \\(\\[{<])(\\\"|\\'{2})\"), r\"\\1 `` \"),\n-        (re.compile(r\"(?i)(\\')(?!re|ve|ll|m|t|s|d)(\\w)\\b\", re.U), r\"\\1 \\2\"),\n+        (re.compile(r\"(?i)(\\')(?!re|ve|ll|m|t|s|d|n)(\\w)\\b\", re.U), r\"\\1 \\2\"),\n     ]\n \n     # Ending quotes.\n", "issue": "Wrong result for word more'n in Word Tokenizer and PTB Tokenizer\nLet sentence: `It's more'n enough`\r\n\r\nIf I'm not mistaken, the PTB Tokenizer should result something like this:\r\n\r\n`[\"It\", \"'s\", \"more\", \"'n\", \"enough\"]`\r\n\r\nBut, it's not. It returns:\r\n\r\n`[\"It\", \"'s\", \"more\", \"'\", \"n\", \"enough\"]`\r\n\r\nSince Word Tokenizer trying to implement PTB contraction, the result should be like that, right?\r\n\r\n**PS**: Word Tokenizer contraction is `mor'n`, while original PTB contraction is `more'n`. Need clarification.\n", "before_files": [{"content": "# Natural Language Toolkit: NLTK's very own tokenizer.\n#\n# Copyright (C) 2001-2020 NLTK Project\n# Author:\n# URL: <http://nltk.sourceforge.net>\n# For license information, see LICENSE.TXT\n\n\nimport re\nfrom nltk.tokenize.api import TokenizerI\n\n\nclass MacIntyreContractions:\n    \"\"\"\n    List of contractions adapted from Robert MacIntyre's tokenizer.\n    \"\"\"\n\n    CONTRACTIONS2 = [\n        r\"(?i)\\b(can)(?#X)(not)\\b\",\n        r\"(?i)\\b(d)(?#X)('ye)\\b\",\n        r\"(?i)\\b(gim)(?#X)(me)\\b\",\n        r\"(?i)\\b(gon)(?#X)(na)\\b\",\n        r\"(?i)\\b(got)(?#X)(ta)\\b\",\n        r\"(?i)\\b(lem)(?#X)(me)\\b\",\n        r\"(?i)\\b(mor)(?#X)('n)\\b\",\n        r\"(?i)\\b(wan)(?#X)(na)\\s\",\n    ]\n    CONTRACTIONS3 = [r\"(?i) ('t)(?#X)(is)\\b\", r\"(?i) ('t)(?#X)(was)\\b\"]\n    CONTRACTIONS4 = [r\"(?i)\\b(whad)(dd)(ya)\\b\", r\"(?i)\\b(wha)(t)(cha)\\b\"]\n\n\nclass NLTKWordTokenizer(TokenizerI):\n    \"\"\"\n    The NLTK tokenizer that has improved upon the TreebankWordTokenizer.\n\n    The tokenizer is \"destructive\" such that the regexes applied will munge the\n    input string to a state beyond re-construction. It is possible to apply\n    `TreebankWordDetokenizer.detokenize` to the tokenized outputs of\n    `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to\n    revert to the original string.\n    \"\"\"\n\n    # Starting quotes.\n    STARTING_QUOTES = [\n        (re.compile(u\"([\u00ab\u201c\u2018\u201e]|[`]+)\", re.U), r\" \\1 \"),\n        (re.compile(r\"^\\\"\"), r\"``\"),\n        (re.compile(r\"(``)\"), r\" \\1 \"),\n        (re.compile(r\"([ \\(\\[{<])(\\\"|\\'{2})\"), r\"\\1 `` \"),\n        (re.compile(r\"(?i)(\\')(?!re|ve|ll|m|t|s|d)(\\w)\\b\", re.U), r\"\\1 \\2\"),\n    ]\n\n    # Ending quotes.\n    ENDING_QUOTES = [\n        (re.compile(u\"([\u00bb\u201d\u2019])\", re.U), r\" \\1 \"),\n        (re.compile(r'\"'), \" '' \"),\n        (re.compile(r\"(\\S)(\\'\\')\"), r\"\\1 \\2 \"),\n        (re.compile(r\"([^' ])('[sS]|'[mM]|'[dD]|') \"), r\"\\1 \\2 \"),\n        (re.compile(r\"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) \"), r\"\\1 \\2 \"),\n    ]\n\n    # For improvements for starting/closing quotes from TreebankWordTokenizer,\n    # see discussion on https://github.com/nltk/nltk/pull/1437\n    # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on\n    # - chervon quotes u'\\xab' and u'\\xbb' .\n    # - unicode quotes u'\\u2018', u'\\u2019', u'\\u201c' and u'\\u201d'\n    # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608\n    # Also, behavior of splitting on clitics now follows Stanford CoreNLP\n    # - clitics covered (?!re|ve|ll|m|t|s|d)(\\w)\\b\n\n    # Punctuation.\n    PUNCTUATION = [\n        (re.compile(r'([^\\.])(\\.)([\\]\\)}>\"\\'' u\"\u00bb\u201d\u2019 \" r\"]*)\\s*$\", re.U), r\"\\1 \\2 \\3 \"),\n        (re.compile(r\"([:,])([^\\d])\"), r\" \\1 \\2\"),\n        (re.compile(r\"([:,])$\"), r\" \\1 \"),\n        (re.compile(r\"\\.{2,}\", re.U), r\" \\g<0> \"), # See https://github.com/nltk/nltk/pull/2322\n        (re.compile(r\"[;@#$%&]\"), r\" \\g<0> \"),\n        (\n            re.compile(r'([^\\.])(\\.)([\\]\\)}>\"\\']*)\\s*$'),\n            r\"\\1 \\2\\3 \",\n        ),  # Handles the final period.\n        (re.compile(r\"[?!]\"), r\" \\g<0> \"),\n        (re.compile(r\"([^'])' \"), r\"\\1 ' \"),\n        (re.compile(r\"[*]\", re.U), r\" \\g<0> \"), # See https://github.com/nltk/nltk/pull/2322\n    ]\n\n    # Pads parentheses\n    PARENS_BRACKETS = (re.compile(r\"[\\]\\[\\(\\)\\{\\}\\<\\>]\"), r\" \\g<0> \")\n\n    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.\n    CONVERT_PARENTHESES = [\n        (re.compile(r\"\\(\"), \"-LRB-\"),\n        (re.compile(r\"\\)\"), \"-RRB-\"),\n        (re.compile(r\"\\[\"), \"-LSB-\"),\n        (re.compile(r\"\\]\"), \"-RSB-\"),\n        (re.compile(r\"\\{\"), \"-LCB-\"),\n        (re.compile(r\"\\}\"), \"-RCB-\"),\n    ]\n\n    DOUBLE_DASHES = (re.compile(r\"--\"), r\" -- \")\n\n    # List of contractions adapted from Robert MacIntyre's tokenizer.\n    _contractions = MacIntyreContractions()\n    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))\n    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))\n\n    def tokenize(self, text, convert_parentheses=False, return_str=False):\n        for regexp, substitution in self.STARTING_QUOTES:\n            text = regexp.sub(substitution, text)\n\n        for regexp, substitution in self.PUNCTUATION:\n            text = regexp.sub(substitution, text)\n\n        # Handles parentheses.\n        regexp, substitution = self.PARENS_BRACKETS\n        text = regexp.sub(substitution, text)\n        # Optionally convert parentheses\n        if convert_parentheses:\n            for regexp, substitution in self.CONVERT_PARENTHESES:\n                text = regexp.sub(substitution, text)\n\n        # Handles double dash.\n        regexp, substitution = self.DOUBLE_DASHES\n        text = regexp.sub(substitution, text)\n\n        # add extra space to make things easier\n        text = \" \" + text + \" \"\n\n        for regexp, substitution in self.ENDING_QUOTES:\n            text = regexp.sub(substitution, text)\n\n        for regexp in self.CONTRACTIONS2:\n            text = regexp.sub(r\" \\1 \\2 \", text)\n        for regexp in self.CONTRACTIONS3:\n            text = regexp.sub(r\" \\1 \\2 \", text)\n\n        # We are not using CONTRACTIONS4 since\n        # they are also commented out in the SED scripts\n        # for regexp in self._contractions.CONTRACTIONS4:\n        #     text = regexp.sub(r' \\1 \\2 \\3 ', text)\n\n        return text if return_str else text.split()\n", "path": "nltk/tokenize/destructive.py"}], "after_files": [{"content": "# Natural Language Toolkit: NLTK's very own tokenizer.\n#\n# Copyright (C) 2001-2020 NLTK Project\n# Author:\n# URL: <http://nltk.sourceforge.net>\n# For license information, see LICENSE.TXT\n\n\nimport re\nfrom nltk.tokenize.api import TokenizerI\n\n\nclass MacIntyreContractions:\n    \"\"\"\n    List of contractions adapted from Robert MacIntyre's tokenizer.\n    \"\"\"\n\n    CONTRACTIONS2 = [\n        r\"(?i)\\b(can)(?#X)(not)\\b\",\n        r\"(?i)\\b(d)(?#X)('ye)\\b\",\n        r\"(?i)\\b(gim)(?#X)(me)\\b\",\n        r\"(?i)\\b(gon)(?#X)(na)\\b\",\n        r\"(?i)\\b(got)(?#X)(ta)\\b\",\n        r\"(?i)\\b(lem)(?#X)(me)\\b\",\n        r\"(?i)\\b(more)(?#X)('n)\\b\",\n        r\"(?i)\\b(wan)(?#X)(na)\\s\",\n    ]\n    CONTRACTIONS3 = [r\"(?i) ('t)(?#X)(is)\\b\", r\"(?i) ('t)(?#X)(was)\\b\"]\n    CONTRACTIONS4 = [r\"(?i)\\b(whad)(dd)(ya)\\b\", r\"(?i)\\b(wha)(t)(cha)\\b\"]\n\n\nclass NLTKWordTokenizer(TokenizerI):\n    \"\"\"\n    The NLTK tokenizer that has improved upon the TreebankWordTokenizer.\n\n    The tokenizer is \"destructive\" such that the regexes applied will munge the\n    input string to a state beyond re-construction. It is possible to apply\n    `TreebankWordDetokenizer.detokenize` to the tokenized outputs of\n    `NLTKDestructiveWordTokenizer.tokenize` but there's no guarantees to\n    revert to the original string.\n    \"\"\"\n\n    # Starting quotes.\n    STARTING_QUOTES = [\n        (re.compile(u\"([\u00ab\u201c\u2018\u201e]|[`]+)\", re.U), r\" \\1 \"),\n        (re.compile(r\"^\\\"\"), r\"``\"),\n        (re.compile(r\"(``)\"), r\" \\1 \"),\n        (re.compile(r\"([ \\(\\[{<])(\\\"|\\'{2})\"), r\"\\1 `` \"),\n        (re.compile(r\"(?i)(\\')(?!re|ve|ll|m|t|s|d|n)(\\w)\\b\", re.U), r\"\\1 \\2\"),\n    ]\n\n    # Ending quotes.\n    ENDING_QUOTES = [\n        (re.compile(u\"([\u00bb\u201d\u2019])\", re.U), r\" \\1 \"),\n        (re.compile(r'\"'), \" '' \"),\n        (re.compile(r\"(\\S)(\\'\\')\"), r\"\\1 \\2 \"),\n        (re.compile(r\"([^' ])('[sS]|'[mM]|'[dD]|') \"), r\"\\1 \\2 \"),\n        (re.compile(r\"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) \"), r\"\\1 \\2 \"),\n    ]\n\n    # For improvements for starting/closing quotes from TreebankWordTokenizer,\n    # see discussion on https://github.com/nltk/nltk/pull/1437\n    # Adding to TreebankWordTokenizer, nltk.word_tokenize now splits on\n    # - chervon quotes u'\\xab' and u'\\xbb' .\n    # - unicode quotes u'\\u2018', u'\\u2019', u'\\u201c' and u'\\u201d'\n    # See https://github.com/nltk/nltk/issues/1995#issuecomment-376741608\n    # Also, behavior of splitting on clitics now follows Stanford CoreNLP\n    # - clitics covered (?!re|ve|ll|m|t|s|d)(\\w)\\b\n\n    # Punctuation.\n    PUNCTUATION = [\n        (re.compile(r'([^\\.])(\\.)([\\]\\)}>\"\\'' u\"\u00bb\u201d\u2019 \" r\"]*)\\s*$\", re.U), r\"\\1 \\2 \\3 \"),\n        (re.compile(r\"([:,])([^\\d])\"), r\" \\1 \\2\"),\n        (re.compile(r\"([:,])$\"), r\" \\1 \"),\n        (re.compile(r\"\\.{2,}\", re.U), r\" \\g<0> \"), # See https://github.com/nltk/nltk/pull/2322\n        (re.compile(r\"[;@#$%&]\"), r\" \\g<0> \"),\n        (\n            re.compile(r'([^\\.])(\\.)([\\]\\)}>\"\\']*)\\s*$'),\n            r\"\\1 \\2\\3 \",\n        ),  # Handles the final period.\n        (re.compile(r\"[?!]\"), r\" \\g<0> \"),\n        (re.compile(r\"([^'])' \"), r\"\\1 ' \"),\n        (re.compile(r\"[*]\", re.U), r\" \\g<0> \"), # See https://github.com/nltk/nltk/pull/2322\n    ]\n\n    # Pads parentheses\n    PARENS_BRACKETS = (re.compile(r\"[\\]\\[\\(\\)\\{\\}\\<\\>]\"), r\" \\g<0> \")\n\n    # Optionally: Convert parentheses, brackets and converts them to PTB symbols.\n    CONVERT_PARENTHESES = [\n        (re.compile(r\"\\(\"), \"-LRB-\"),\n        (re.compile(r\"\\)\"), \"-RRB-\"),\n        (re.compile(r\"\\[\"), \"-LSB-\"),\n        (re.compile(r\"\\]\"), \"-RSB-\"),\n        (re.compile(r\"\\{\"), \"-LCB-\"),\n        (re.compile(r\"\\}\"), \"-RCB-\"),\n    ]\n\n    DOUBLE_DASHES = (re.compile(r\"--\"), r\" -- \")\n\n    # List of contractions adapted from Robert MacIntyre's tokenizer.\n    _contractions = MacIntyreContractions()\n    CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))\n    CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))\n\n    def tokenize(self, text, convert_parentheses=False, return_str=False):\n        for regexp, substitution in self.STARTING_QUOTES:\n            text = regexp.sub(substitution, text)\n\n        for regexp, substitution in self.PUNCTUATION:\n            text = regexp.sub(substitution, text)\n\n        # Handles parentheses.\n        regexp, substitution = self.PARENS_BRACKETS\n        text = regexp.sub(substitution, text)\n        # Optionally convert parentheses\n        if convert_parentheses:\n            for regexp, substitution in self.CONVERT_PARENTHESES:\n                text = regexp.sub(substitution, text)\n\n        # Handles double dash.\n        regexp, substitution = self.DOUBLE_DASHES\n        text = regexp.sub(substitution, text)\n\n        # add extra space to make things easier\n        text = \" \" + text + \" \"\n\n        for regexp, substitution in self.ENDING_QUOTES:\n            text = regexp.sub(substitution, text)\n\n        for regexp in self.CONTRACTIONS2:\n            text = regexp.sub(r\" \\1 \\2 \", text)\n        for regexp in self.CONTRACTIONS3:\n            text = regexp.sub(r\" \\1 \\2 \", text)\n\n        # We are not using CONTRACTIONS4 since\n        # they are also commented out in the SED scripts\n        # for regexp in self._contractions.CONTRACTIONS4:\n        #     text = regexp.sub(r' \\1 \\2 \\3 ', text)\n\n        return text if return_str else text.split()\n", "path": "nltk/tokenize/destructive.py"}]}