We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Simplify AutoStrEnum definition in Jupyter magic's argument parser
The current definition of [AutoStrEnum](https://github.com/googleapis/python-bigquery/blob/0337ea0bde966c3ccb94960493a6fa6f2bee49b4/google/cloud/bigquery/magics/line_arg_parser/lexer.py#L139-L167) is rather complicated. With the library now only supporting Python 3.6+, we can simplify it and just use the `_generate_next_value_()` hook.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `google/cloud/bigquery/magics/line_arg_parser/lexer.py`
Content:
```
1 # Copyright 2020 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from collections import namedtuple
16 from collections import OrderedDict
17 import itertools
18 import re
19 
20 import enum
21 
22 
23 Token = namedtuple("Token", ("type_", "lexeme", "pos"))
24 StateTransition = namedtuple("StateTransition", ("new_state", "total_offset"))
25 
26 # Pattern matching is done with regexes, and the order in which the token patterns are
27 # defined is important.
28 #
29 # Suppose we had the following token definitions:
30 #  * INT - a token matching integers,
31 #  * FLOAT - a token matching floating point numbers,
32 #  * DOT - a token matching a single literal dot character, i.e. "."
33 #
34 # The FLOAT token would have to be defined first, since we would want the input "1.23"
35 # to be tokenized as a single FLOAT token, and *not* three tokens (INT, DOT, INT).
36 #
37 # Sometimes, however, different tokens match too similar patterns, and it is not
38 # possible to define them in order that would avoid any ambiguity. One such case are
39 # the OPT_VAL and PY_NUMBER tokens, as both can match an integer literal, say "42".
40 #
41 # In order to avoid the dilemmas, the lexer implements a concept of STATES. States are
42 # used to split token definitions into subgroups, and in each lexer state only a single
43 # subgroup is used for tokenizing the input. Lexer states can therefore be though of as
44 # token namespaces.
45 #
46 # For example, while parsing the value of the "--params" option, we do not want to
47 # "recognize" it as a single OPT_VAL token, but instead want to parse it as a Python
48 # dictionary and verify its syntactial correctness. On the other hand, while parsing
49 # the value of an option other than "--params", we do not really care about its
50 # structure, and thus do not want to use any of the "Python tokens" for pattern matching.
51 #
52 # Since token definition order is important, an OrderedDict is needed with tightly
53 # controlled member definitions (i.e. passed as a sequence, and *not* via kwargs).
54 token_types = OrderedDict(
55     [
56         (
57             "state_parse_pos_args",
58             OrderedDict(
59                 [
60                     (
61                         "GOTO_PARSE_NON_PARAMS_OPTIONS",
62                         r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))",  # double dash - starting the options list
63                     ),
64                     (
65                         "DEST_VAR",
66                         r"(?P<DEST_VAR>[^\d\W]\w*)",  # essentially a Python ID
67                     ),
68                 ]
69             ),
70         ),
71         (
72             "state_parse_non_params_options",
73             OrderedDict(
74                 [
75                     (
76                         "GOTO_PARSE_PARAMS_OPTION",
77                         r"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\s|=|--|$)))",  # the --params option
78                     ),
79                     ("OPTION_SPEC", r"(?P<OPTION_SPEC>--\w+)"),
80                     ("OPTION_EQ", r"(?P<OPTION_EQ>=)"),
81                     ("OPT_VAL", r"(?P<OPT_VAL>\S+?(?=\s|--|$))"),
82                 ]
83             ),
84         ),
85         (
86             "state_parse_params_option",
87             OrderedDict(
88                 [
89                     (
90                         "PY_STRING",
91                         r"(?P<PY_STRING>(?:{})|(?:{}))".format(
92                             r"'(?:[^'\\]|\.)*'",
93                             r'"(?:[^"\\]|\.)*"',  # single and double quoted strings
94                         ),
95                     ),
96                     ("PARAMS_OPT_SPEC", r"(?P<PARAMS_OPT_SPEC>--params(?=\s|=|--|$))"),
97                     ("PARAMS_OPT_EQ", r"(?P<PARAMS_OPT_EQ>=)"),
98                     (
99                         "GOTO_PARSE_NON_PARAMS_OPTIONS",
100                         r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\w+))",  # found another option spec
101                     ),
102                     ("PY_BOOL", r"(?P<PY_BOOL>True|False)"),
103                     ("DOLLAR_PY_ID", r"(?P<DOLLAR_PY_ID>\$[^\d\W]\w*)"),
104                     (
105                         "PY_NUMBER",
106                         r"(?P<PY_NUMBER>-?[1-9]\d*(?:\.\d+)?(:?[e|E][+-]?\d+)?)",
107                     ),
108                     ("SQUOTE", r"(?P<SQUOTE>')"),
109                     ("DQUOTE", r'(?P<DQUOTE>")'),
110                     ("COLON", r"(?P<COLON>:)"),
111                     ("COMMA", r"(?P<COMMA>,)"),
112                     ("LCURL", r"(?P<LCURL>\{)"),
113                     ("RCURL", r"(?P<RCURL>})"),
114                     ("LSQUARE", r"(?P<LSQUARE>\[)"),
115                     ("RSQUARE", r"(?P<RSQUARE>])"),
116                     ("LPAREN", r"(?P<LPAREN>\()"),
117                     ("RPAREN", r"(?P<RPAREN>\))"),
118                 ]
119             ),
120         ),
121         (
122             "common",
123             OrderedDict(
124                 [
125                     ("WS", r"(?P<WS>\s+)"),
126                     ("EOL", r"(?P<EOL>$)"),
127                     (
128                         # anything not a whitespace or matched by something else
129                         "UNKNOWN",
130                         r"(?P<UNKNOWN>\S+)",
131                     ),
132                 ]
133             ),
134         ),
135     ]
136 )
137 
138 
139 # The _generate_next_value_() enum hook is only available in Python 3.6+, thus we
140 # need to do some acrobatics to implement an "auto str enum" base class. Implementation
141 # based on the recipe provided by the very author of the Enum library:
142 # https://stackoverflow.com/a/32313954/5040035
143 class StrEnumMeta(enum.EnumMeta):
144     @classmethod
145     def __prepare__(metacls, name, bases, **kwargs):
146         # Having deterministic enum members definition order is nice.
147         return OrderedDict()
148 
149     def __new__(metacls, name, bases, oldclassdict):
150         # Scan through the declared enum members and convert any value that is a plain
151         # empty tuple into a `str` of the name instead.
152         newclassdict = enum._EnumDict()
153         for key, val in oldclassdict.items():
154             if val == ():
155                 val = key
156             newclassdict[key] = val
157         return super(StrEnumMeta, metacls).__new__(metacls, name, bases, newclassdict)
158 
159 
160 # The @six.add_metaclass decorator does not work, Enum complains about _sunder_ names,
161 # and we cannot use class syntax directly, because the Python 3 version would cause
162 # a syntax error under Python 2.
163 AutoStrEnum = StrEnumMeta(
164     "AutoStrEnum",
165     (str, enum.Enum),
166     {"__doc__": "Base enum class for for name=value str enums."},
167 )
168 
169 TokenType = AutoStrEnum(
170     "TokenType",
171     [
172         (name, name)
173         for name in itertools.chain.from_iterable(token_types.values())
174         if not name.startswith("GOTO_")
175     ],
176 )
177 
178 
179 class LexerState(AutoStrEnum):
180     PARSE_POS_ARGS = ()  # parsing positional arguments
181     PARSE_NON_PARAMS_OPTIONS = ()  # parsing options other than "--params"
182     PARSE_PARAMS_OPTION = ()  # parsing the "--params" option
183     STATE_END = ()
184 
185 
186 class Lexer(object):
187     """Lexical analyzer for tokenizing the cell magic input line."""
188 
189     _GRAND_PATTERNS = {
190         LexerState.PARSE_POS_ARGS: re.compile(
191             "|".join(
192                 itertools.chain(
193                     token_types["state_parse_pos_args"].values(),
194                     token_types["common"].values(),
195                 )
196             )
197         ),
198         LexerState.PARSE_NON_PARAMS_OPTIONS: re.compile(
199             "|".join(
200                 itertools.chain(
201                     token_types["state_parse_non_params_options"].values(),
202                     token_types["common"].values(),
203                 )
204             )
205         ),
206         LexerState.PARSE_PARAMS_OPTION: re.compile(
207             "|".join(
208                 itertools.chain(
209                     token_types["state_parse_params_option"].values(),
210                     token_types["common"].values(),
211                 )
212             )
213         ),
214     }
215 
216     def __init__(self, input_text):
217         self._text = input_text
218 
219     def __iter__(self):
220         # Since re.scanner does not seem to support manipulating inner scanner states,
221         # we need to implement lexer state transitions manually using special
222         # non-capturing lookahead token patterns to signal when a state transition
223         # should be made.
224         # Since we don't have "nested" states, we don't really need a stack and
225         # this simple mechanism is sufficient.
226         state = LexerState.PARSE_POS_ARGS
227         offset = 0  # the number of characters processed so far
228 
229         while state != LexerState.STATE_END:
230             token_stream = self._find_state_tokens(state, offset)
231 
232             for maybe_token in token_stream:  # pragma: NO COVER
233                 if isinstance(maybe_token, StateTransition):
234                     state = maybe_token.new_state
235                     offset = maybe_token.total_offset
236                     break
237 
238                 if maybe_token.type_ != TokenType.WS:
239                     yield maybe_token
240 
241                 if maybe_token.type_ == TokenType.EOL:
242                     state = LexerState.STATE_END
243                     break
244 
245     def _find_state_tokens(self, state, current_offset):
246         """Scan the input for current state's tokens starting at ``current_offset``.
247 
248         Args:
249             state (LexerState): The current lexer state.
250             current_offset (int): The offset in the input text, i.e. the number
251                 of characters already scanned so far.
252 
253         Yields:
254             The next ``Token`` or ``StateTransition`` instance.
255         """
256         pattern = self._GRAND_PATTERNS[state]
257         scanner = pattern.finditer(self._text, current_offset)
258 
259         for match in scanner:  # pragma: NO COVER
260             token_type = match.lastgroup
261 
262             if token_type.startswith("GOTO_"):
263                 yield StateTransition(
264                     new_state=getattr(LexerState, token_type[5:]),  # w/o "GOTO_" prefix
265                     total_offset=match.start(),
266                 )
267 
268             yield Token(token_type, match.group(), match.start())
269
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/google/cloud/bigquery/magics/line_arg_parser/lexer.py b/google/cloud/bigquery/magics/line_arg_parser/lexer.py
--- a/google/cloud/bigquery/magics/line_arg_parser/lexer.py
+++ b/google/cloud/bigquery/magics/line_arg_parser/lexer.py
@@ -136,40 +136,17 @@
 )
 
 
-# The _generate_next_value_() enum hook is only available in Python 3.6+, thus we
-# need to do some acrobatics to implement an "auto str enum" base class. Implementation
-# based on the recipe provided by the very author of the Enum library:
-# https://stackoverflow.com/a/32313954/5040035
-class StrEnumMeta(enum.EnumMeta):
-    @classmethod
-    def __prepare__(metacls, name, bases, **kwargs):
-        # Having deterministic enum members definition order is nice.
-        return OrderedDict()
+class AutoStrEnum(str, enum.Enum):
+    """Base enum class for for name=value str enums."""
 
-    def __new__(metacls, name, bases, oldclassdict):
-        # Scan through the declared enum members and convert any value that is a plain
-        # empty tuple into a `str` of the name instead.
-        newclassdict = enum._EnumDict()
-        for key, val in oldclassdict.items():
-            if val == ():
-                val = key
-            newclassdict[key] = val
-        return super(StrEnumMeta, metacls).__new__(metacls, name, bases, newclassdict)
+    def _generate_next_value_(name, start, count, last_values):
+        return name
 
 
-# The @six.add_metaclass decorator does not work, Enum complains about _sunder_ names,
-# and we cannot use class syntax directly, because the Python 3 version would cause
-# a syntax error under Python 2.
-AutoStrEnum = StrEnumMeta(
-    "AutoStrEnum",
-    (str, enum.Enum),
-    {"__doc__": "Base enum class for for name=value str enums."},
-)
-
 TokenType = AutoStrEnum(
     "TokenType",
     [
-        (name, name)
+        (name, enum.auto())
         for name in itertools.chain.from_iterable(token_types.values())
         if not name.startswith("GOTO_")
     ],
@@ -177,10 +154,10 @@
 
 
 class LexerState(AutoStrEnum):
-    PARSE_POS_ARGS = ()  # parsing positional arguments
-    PARSE_NON_PARAMS_OPTIONS = ()  # parsing options other than "--params"
-    PARSE_PARAMS_OPTION = ()  # parsing the "--params" option
-    STATE_END = ()
+    PARSE_POS_ARGS = enum.auto()  # parsing positional arguments
+    PARSE_NON_PARAMS_OPTIONS = enum.auto()  # parsing options other than "--params"
+    PARSE_PARAMS_OPTION = enum.auto()  # parsing the "--params" option
+    STATE_END = enum.auto()
 
 
 class Lexer(object):

{"golden_diff": "diff --git a/google/cloud/bigquery/magics/line_arg_parser/lexer.py b/google/cloud/bigquery/magics/line_arg_parser/lexer.py\n--- a/google/cloud/bigquery/magics/line_arg_parser/lexer.py\n+++ b/google/cloud/bigquery/magics/line_arg_parser/lexer.py\n@@ -136,40 +136,17 @@\n )\n \n \n-# The _generate_next_value_() enum hook is only available in Python 3.6+, thus we\n-# need to do some acrobatics to implement an \"auto str enum\" base class. Implementation\n-# based on the recipe provided by the very author of the Enum library:\n-# https://stackoverflow.com/a/32313954/5040035\n-class StrEnumMeta(enum.EnumMeta):\n-    @classmethod\n-    def __prepare__(metacls, name, bases, **kwargs):\n-        # Having deterministic enum members definition order is nice.\n-        return OrderedDict()\n+class AutoStrEnum(str, enum.Enum):\n+    \"\"\"Base enum class for for name=value str enums.\"\"\"\n \n-    def __new__(metacls, name, bases, oldclassdict):\n-        # Scan through the declared enum members and convert any value that is a plain\n-        # empty tuple into a `str` of the name instead.\n-        newclassdict = enum._EnumDict()\n-        for key, val in oldclassdict.items():\n-            if val == ():\n-                val = key\n-            newclassdict[key] = val\n-        return super(StrEnumMeta, metacls).__new__(metacls, name, bases, newclassdict)\n+    def _generate_next_value_(name, start, count, last_values):\n+        return name\n \n \n-# The @six.add_metaclass decorator does not work, Enum complains about _sunder_ names,\n-# and we cannot use class syntax directly, because the Python 3 version would cause\n-# a syntax error under Python 2.\n-AutoStrEnum = StrEnumMeta(\n-    \"AutoStrEnum\",\n-    (str, enum.Enum),\n-    {\"__doc__\": \"Base enum class for for name=value str enums.\"},\n-)\n-\n TokenType = AutoStrEnum(\n     \"TokenType\",\n     [\n-        (name, name)\n+        (name, enum.auto())\n         for name in itertools.chain.from_iterable(token_types.values())\n         if not name.startswith(\"GOTO_\")\n     ],\n@@ -177,10 +154,10 @@\n \n \n class LexerState(AutoStrEnum):\n-    PARSE_POS_ARGS = ()  # parsing positional arguments\n-    PARSE_NON_PARAMS_OPTIONS = ()  # parsing options other than \"--params\"\n-    PARSE_PARAMS_OPTION = ()  # parsing the \"--params\" option\n-    STATE_END = ()\n+    PARSE_POS_ARGS = enum.auto()  # parsing positional arguments\n+    PARSE_NON_PARAMS_OPTIONS = enum.auto()  # parsing options other than \"--params\"\n+    PARSE_PARAMS_OPTION = enum.auto()  # parsing the \"--params\" option\n+    STATE_END = enum.auto()\n \n \n class Lexer(object):\n", "issue": "Simplify AutoStrEnum definition in Jupyter magic's argument parser\nThe current definition of [AutoStrEnum](https://github.com/googleapis/python-bigquery/blob/0337ea0bde966c3ccb94960493a6fa6f2bee49b4/google/cloud/bigquery/magics/line_arg_parser/lexer.py#L139-L167) is rather complicated. With the library now only supporting Python 3.6+, we can simplify it and just use the `_generate_next_value_()` hook.\n", "before_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom collections import namedtuple\nfrom collections import OrderedDict\nimport itertools\nimport re\n\nimport enum\n\n\nToken = namedtuple(\"Token\", (\"type_\", \"lexeme\", \"pos\"))\nStateTransition = namedtuple(\"StateTransition\", (\"new_state\", \"total_offset\"))\n\n# Pattern matching is done with regexes, and the order in which the token patterns are\n# defined is important.\n#\n# Suppose we had the following token definitions:\n#  * INT - a token matching integers,\n#  * FLOAT - a token matching floating point numbers,\n#  * DOT - a token matching a single literal dot character, i.e. \".\"\n#\n# The FLOAT token would have to be defined first, since we would want the input \"1.23\"\n# to be tokenized as a single FLOAT token, and *not* three tokens (INT, DOT, INT).\n#\n# Sometimes, however, different tokens match too similar patterns, and it is not\n# possible to define them in order that would avoid any ambiguity. One such case are\n# the OPT_VAL and PY_NUMBER tokens, as both can match an integer literal, say \"42\".\n#\n# In order to avoid the dilemmas, the lexer implements a concept of STATES. States are\n# used to split token definitions into subgroups, and in each lexer state only a single\n# subgroup is used for tokenizing the input. Lexer states can therefore be though of as\n# token namespaces.\n#\n# For example, while parsing the value of the \"--params\" option, we do not want to\n# \"recognize\" it as a single OPT_VAL token, but instead want to parse it as a Python\n# dictionary and verify its syntactial correctness. On the other hand, while parsing\n# the value of an option other than \"--params\", we do not really care about its\n# structure, and thus do not want to use any of the \"Python tokens\" for pattern matching.\n#\n# Since token definition order is important, an OrderedDict is needed with tightly\n# controlled member definitions (i.e. passed as a sequence, and *not* via kwargs).\ntoken_types = OrderedDict(\n    [\n        (\n            \"state_parse_pos_args\",\n            OrderedDict(\n                [\n                    (\n                        \"GOTO_PARSE_NON_PARAMS_OPTIONS\",\n                        r\"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))\",  # double dash - starting the options list\n                    ),\n                    (\n                        \"DEST_VAR\",\n                        r\"(?P<DEST_VAR>[^\\d\\W]\\w*)\",  # essentially a Python ID\n                    ),\n                ]\n            ),\n        ),\n        (\n            \"state_parse_non_params_options\",\n            OrderedDict(\n                [\n                    (\n                        \"GOTO_PARSE_PARAMS_OPTION\",\n                        r\"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\\s|=|--|$)))\",  # the --params option\n                    ),\n                    (\"OPTION_SPEC\", r\"(?P<OPTION_SPEC>--\\w+)\"),\n                    (\"OPTION_EQ\", r\"(?P<OPTION_EQ>=)\"),\n                    (\"OPT_VAL\", r\"(?P<OPT_VAL>\\S+?(?=\\s|--|$))\"),\n                ]\n            ),\n        ),\n        (\n            \"state_parse_params_option\",\n            OrderedDict(\n                [\n                    (\n                        \"PY_STRING\",\n                        r\"(?P<PY_STRING>(?:{})|(?:{}))\".format(\n                            r\"'(?:[^'\\\\]|\\.)*'\",\n                            r'\"(?:[^\"\\\\]|\\.)*\"',  # single and double quoted strings\n                        ),\n                    ),\n                    (\"PARAMS_OPT_SPEC\", r\"(?P<PARAMS_OPT_SPEC>--params(?=\\s|=|--|$))\"),\n                    (\"PARAMS_OPT_EQ\", r\"(?P<PARAMS_OPT_EQ>=)\"),\n                    (\n                        \"GOTO_PARSE_NON_PARAMS_OPTIONS\",\n                        r\"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\\w+))\",  # found another option spec\n                    ),\n                    (\"PY_BOOL\", r\"(?P<PY_BOOL>True|False)\"),\n                    (\"DOLLAR_PY_ID\", r\"(?P<DOLLAR_PY_ID>\\$[^\\d\\W]\\w*)\"),\n                    (\n                        \"PY_NUMBER\",\n                        r\"(?P<PY_NUMBER>-?[1-9]\\d*(?:\\.\\d+)?(:?[e|E][+-]?\\d+)?)\",\n                    ),\n                    (\"SQUOTE\", r\"(?P<SQUOTE>')\"),\n                    (\"DQUOTE\", r'(?P<DQUOTE>\")'),\n                    (\"COLON\", r\"(?P<COLON>:)\"),\n                    (\"COMMA\", r\"(?P<COMMA>,)\"),\n                    (\"LCURL\", r\"(?P<LCURL>\\{)\"),\n                    (\"RCURL\", r\"(?P<RCURL>})\"),\n                    (\"LSQUARE\", r\"(?P<LSQUARE>\\[)\"),\n                    (\"RSQUARE\", r\"(?P<RSQUARE>])\"),\n                    (\"LPAREN\", r\"(?P<LPAREN>\\()\"),\n                    (\"RPAREN\", r\"(?P<RPAREN>\\))\"),\n                ]\n            ),\n        ),\n        (\n            \"common\",\n            OrderedDict(\n                [\n                    (\"WS\", r\"(?P<WS>\\s+)\"),\n                    (\"EOL\", r\"(?P<EOL>$)\"),\n                    (\n                        # anything not a whitespace or matched by something else\n                        \"UNKNOWN\",\n                        r\"(?P<UNKNOWN>\\S+)\",\n                    ),\n                ]\n            ),\n        ),\n    ]\n)\n\n\n# The _generate_next_value_() enum hook is only available in Python 3.6+, thus we\n# need to do some acrobatics to implement an \"auto str enum\" base class. Implementation\n# based on the recipe provided by the very author of the Enum library:\n# https://stackoverflow.com/a/32313954/5040035\nclass StrEnumMeta(enum.EnumMeta):\n    @classmethod\n    def __prepare__(metacls, name, bases, **kwargs):\n        # Having deterministic enum members definition order is nice.\n        return OrderedDict()\n\n    def __new__(metacls, name, bases, oldclassdict):\n        # Scan through the declared enum members and convert any value that is a plain\n        # empty tuple into a `str` of the name instead.\n        newclassdict = enum._EnumDict()\n        for key, val in oldclassdict.items():\n            if val == ():\n                val = key\n            newclassdict[key] = val\n        return super(StrEnumMeta, metacls).__new__(metacls, name, bases, newclassdict)\n\n\n# The @six.add_metaclass decorator does not work, Enum complains about _sunder_ names,\n# and we cannot use class syntax directly, because the Python 3 version would cause\n# a syntax error under Python 2.\nAutoStrEnum = StrEnumMeta(\n    \"AutoStrEnum\",\n    (str, enum.Enum),\n    {\"__doc__\": \"Base enum class for for name=value str enums.\"},\n)\n\nTokenType = AutoStrEnum(\n    \"TokenType\",\n    [\n        (name, name)\n        for name in itertools.chain.from_iterable(token_types.values())\n        if not name.startswith(\"GOTO_\")\n    ],\n)\n\n\nclass LexerState(AutoStrEnum):\n    PARSE_POS_ARGS = ()  # parsing positional arguments\n    PARSE_NON_PARAMS_OPTIONS = ()  # parsing options other than \"--params\"\n    PARSE_PARAMS_OPTION = ()  # parsing the \"--params\" option\n    STATE_END = ()\n\n\nclass Lexer(object):\n    \"\"\"Lexical analyzer for tokenizing the cell magic input line.\"\"\"\n\n    _GRAND_PATTERNS = {\n        LexerState.PARSE_POS_ARGS: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_pos_args\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n        LexerState.PARSE_NON_PARAMS_OPTIONS: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_non_params_options\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n        LexerState.PARSE_PARAMS_OPTION: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_params_option\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n    }\n\n    def __init__(self, input_text):\n        self._text = input_text\n\n    def __iter__(self):\n        # Since re.scanner does not seem to support manipulating inner scanner states,\n        # we need to implement lexer state transitions manually using special\n        # non-capturing lookahead token patterns to signal when a state transition\n        # should be made.\n        # Since we don't have \"nested\" states, we don't really need a stack and\n        # this simple mechanism is sufficient.\n        state = LexerState.PARSE_POS_ARGS\n        offset = 0  # the number of characters processed so far\n\n        while state != LexerState.STATE_END:\n            token_stream = self._find_state_tokens(state, offset)\n\n            for maybe_token in token_stream:  # pragma: NO COVER\n                if isinstance(maybe_token, StateTransition):\n                    state = maybe_token.new_state\n                    offset = maybe_token.total_offset\n                    break\n\n                if maybe_token.type_ != TokenType.WS:\n                    yield maybe_token\n\n                if maybe_token.type_ == TokenType.EOL:\n                    state = LexerState.STATE_END\n                    break\n\n    def _find_state_tokens(self, state, current_offset):\n        \"\"\"Scan the input for current state's tokens starting at ``current_offset``.\n\n        Args:\n            state (LexerState): The current lexer state.\n            current_offset (int): The offset in the input text, i.e. the number\n                of characters already scanned so far.\n\n        Yields:\n            The next ``Token`` or ``StateTransition`` instance.\n        \"\"\"\n        pattern = self._GRAND_PATTERNS[state]\n        scanner = pattern.finditer(self._text, current_offset)\n\n        for match in scanner:  # pragma: NO COVER\n            token_type = match.lastgroup\n\n            if token_type.startswith(\"GOTO_\"):\n                yield StateTransition(\n                    new_state=getattr(LexerState, token_type[5:]),  # w/o \"GOTO_\" prefix\n                    total_offset=match.start(),\n                )\n\n            yield Token(token_type, match.group(), match.start())\n", "path": "google/cloud/bigquery/magics/line_arg_parser/lexer.py"}], "after_files": [{"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom collections import namedtuple\nfrom collections import OrderedDict\nimport itertools\nimport re\n\nimport enum\n\n\nToken = namedtuple(\"Token\", (\"type_\", \"lexeme\", \"pos\"))\nStateTransition = namedtuple(\"StateTransition\", (\"new_state\", \"total_offset\"))\n\n# Pattern matching is done with regexes, and the order in which the token patterns are\n# defined is important.\n#\n# Suppose we had the following token definitions:\n#  * INT - a token matching integers,\n#  * FLOAT - a token matching floating point numbers,\n#  * DOT - a token matching a single literal dot character, i.e. \".\"\n#\n# The FLOAT token would have to be defined first, since we would want the input \"1.23\"\n# to be tokenized as a single FLOAT token, and *not* three tokens (INT, DOT, INT).\n#\n# Sometimes, however, different tokens match too similar patterns, and it is not\n# possible to define them in order that would avoid any ambiguity. One such case are\n# the OPT_VAL and PY_NUMBER tokens, as both can match an integer literal, say \"42\".\n#\n# In order to avoid the dilemmas, the lexer implements a concept of STATES. States are\n# used to split token definitions into subgroups, and in each lexer state only a single\n# subgroup is used for tokenizing the input. Lexer states can therefore be though of as\n# token namespaces.\n#\n# For example, while parsing the value of the \"--params\" option, we do not want to\n# \"recognize\" it as a single OPT_VAL token, but instead want to parse it as a Python\n# dictionary and verify its syntactial correctness. On the other hand, while parsing\n# the value of an option other than \"--params\", we do not really care about its\n# structure, and thus do not want to use any of the \"Python tokens\" for pattern matching.\n#\n# Since token definition order is important, an OrderedDict is needed with tightly\n# controlled member definitions (i.e. passed as a sequence, and *not* via kwargs).\ntoken_types = OrderedDict(\n    [\n        (\n            \"state_parse_pos_args\",\n            OrderedDict(\n                [\n                    (\n                        \"GOTO_PARSE_NON_PARAMS_OPTIONS\",\n                        r\"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))\",  # double dash - starting the options list\n                    ),\n                    (\n                        \"DEST_VAR\",\n                        r\"(?P<DEST_VAR>[^\\d\\W]\\w*)\",  # essentially a Python ID\n                    ),\n                ]\n            ),\n        ),\n        (\n            \"state_parse_non_params_options\",\n            OrderedDict(\n                [\n                    (\n                        \"GOTO_PARSE_PARAMS_OPTION\",\n                        r\"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\\s|=|--|$)))\",  # the --params option\n                    ),\n                    (\"OPTION_SPEC\", r\"(?P<OPTION_SPEC>--\\w+)\"),\n                    (\"OPTION_EQ\", r\"(?P<OPTION_EQ>=)\"),\n                    (\"OPT_VAL\", r\"(?P<OPT_VAL>\\S+?(?=\\s|--|$))\"),\n                ]\n            ),\n        ),\n        (\n            \"state_parse_params_option\",\n            OrderedDict(\n                [\n                    (\n                        \"PY_STRING\",\n                        r\"(?P<PY_STRING>(?:{})|(?:{}))\".format(\n                            r\"'(?:[^'\\\\]|\\.)*'\",\n                            r'\"(?:[^\"\\\\]|\\.)*\"',  # single and double quoted strings\n                        ),\n                    ),\n                    (\"PARAMS_OPT_SPEC\", r\"(?P<PARAMS_OPT_SPEC>--params(?=\\s|=|--|$))\"),\n                    (\"PARAMS_OPT_EQ\", r\"(?P<PARAMS_OPT_EQ>=)\"),\n                    (\n                        \"GOTO_PARSE_NON_PARAMS_OPTIONS\",\n                        r\"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\\w+))\",  # found another option spec\n                    ),\n                    (\"PY_BOOL\", r\"(?P<PY_BOOL>True|False)\"),\n                    (\"DOLLAR_PY_ID\", r\"(?P<DOLLAR_PY_ID>\\$[^\\d\\W]\\w*)\"),\n                    (\n                        \"PY_NUMBER\",\n                        r\"(?P<PY_NUMBER>-?[1-9]\\d*(?:\\.\\d+)?(:?[e|E][+-]?\\d+)?)\",\n                    ),\n                    (\"SQUOTE\", r\"(?P<SQUOTE>')\"),\n                    (\"DQUOTE\", r'(?P<DQUOTE>\")'),\n                    (\"COLON\", r\"(?P<COLON>:)\"),\n                    (\"COMMA\", r\"(?P<COMMA>,)\"),\n                    (\"LCURL\", r\"(?P<LCURL>\\{)\"),\n                    (\"RCURL\", r\"(?P<RCURL>})\"),\n                    (\"LSQUARE\", r\"(?P<LSQUARE>\\[)\"),\n                    (\"RSQUARE\", r\"(?P<RSQUARE>])\"),\n                    (\"LPAREN\", r\"(?P<LPAREN>\\()\"),\n                    (\"RPAREN\", r\"(?P<RPAREN>\\))\"),\n                ]\n            ),\n        ),\n        (\n            \"common\",\n            OrderedDict(\n                [\n                    (\"WS\", r\"(?P<WS>\\s+)\"),\n                    (\"EOL\", r\"(?P<EOL>$)\"),\n                    (\n                        # anything not a whitespace or matched by something else\n                        \"UNKNOWN\",\n                        r\"(?P<UNKNOWN>\\S+)\",\n                    ),\n                ]\n            ),\n        ),\n    ]\n)\n\n\nclass AutoStrEnum(str, enum.Enum):\n    \"\"\"Base enum class for for name=value str enums.\"\"\"\n\n    def _generate_next_value_(name, start, count, last_values):\n        return name\n\n\nTokenType = AutoStrEnum(\n    \"TokenType\",\n    [\n        (name, enum.auto())\n        for name in itertools.chain.from_iterable(token_types.values())\n        if not name.startswith(\"GOTO_\")\n    ],\n)\n\n\nclass LexerState(AutoStrEnum):\n    PARSE_POS_ARGS = enum.auto()  # parsing positional arguments\n    PARSE_NON_PARAMS_OPTIONS = enum.auto()  # parsing options other than \"--params\"\n    PARSE_PARAMS_OPTION = enum.auto()  # parsing the \"--params\" option\n    STATE_END = enum.auto()\n\n\nclass Lexer(object):\n    \"\"\"Lexical analyzer for tokenizing the cell magic input line.\"\"\"\n\n    _GRAND_PATTERNS = {\n        LexerState.PARSE_POS_ARGS: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_pos_args\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n        LexerState.PARSE_NON_PARAMS_OPTIONS: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_non_params_options\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n        LexerState.PARSE_PARAMS_OPTION: re.compile(\n            \"|\".join(\n                itertools.chain(\n                    token_types[\"state_parse_params_option\"].values(),\n                    token_types[\"common\"].values(),\n                )\n            )\n        ),\n    }\n\n    def __init__(self, input_text):\n        self._text = input_text\n\n    def __iter__(self):\n        # Since re.scanner does not seem to support manipulating inner scanner states,\n        # we need to implement lexer state transitions manually using special\n        # non-capturing lookahead token patterns to signal when a state transition\n        # should be made.\n        # Since we don't have \"nested\" states, we don't really need a stack and\n        # this simple mechanism is sufficient.\n        state = LexerState.PARSE_POS_ARGS\n        offset = 0  # the number of characters processed so far\n\n        while state != LexerState.STATE_END:\n            token_stream = self._find_state_tokens(state, offset)\n\n            for maybe_token in token_stream:  # pragma: NO COVER\n                if isinstance(maybe_token, StateTransition):\n                    state = maybe_token.new_state\n                    offset = maybe_token.total_offset\n                    break\n\n                if maybe_token.type_ != TokenType.WS:\n                    yield maybe_token\n\n                if maybe_token.type_ == TokenType.EOL:\n                    state = LexerState.STATE_END\n                    break\n\n    def _find_state_tokens(self, state, current_offset):\n        \"\"\"Scan the input for current state's tokens starting at ``current_offset``.\n\n        Args:\n            state (LexerState): The current lexer state.\n            current_offset (int): The offset in the input text, i.e. the number\n                of characters already scanned so far.\n\n        Yields:\n            The next ``Token`` or ``StateTransition`` instance.\n        \"\"\"\n        pattern = self._GRAND_PATTERNS[state]\n        scanner = pattern.finditer(self._text, current_offset)\n\n        for match in scanner:  # pragma: NO COVER\n            token_type = match.lastgroup\n\n            if token_type.startswith(\"GOTO_\"):\n                yield StateTransition(\n                    new_state=getattr(LexerState, token_type[5:]),  # w/o \"GOTO_\" prefix\n                    total_offset=match.start(),\n                )\n\n            yield Token(token_type, match.group(), match.start())\n", "path": "google/cloud/bigquery/magics/line_arg_parser/lexer.py"}]}