We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Identifier quoting clashes with fully-qualified names
The point of identifier quoting is that it allows using non-alphanumeric characters or reserved keywords as identifiers. So consider the following:
```
CREATE TYPE `foo::Bar`;
```
It will be interpreted as a single identifier, which then gets prefixed with the `default` module name to create a fully-qualified name with module `default` and name `foo::Bar`. Unfortunately, during processing it gets normalized into a single string `"default::foo::Bar"`, which then further down the line gets (mis-)interpreted as module `default::foo` and name `Bar`.
```
edgedb> CREATE TYPE `foo::Bar`;
CREATE
edgedb> WITH MODULE schema
....... SELECT ObjectType {name};
UnknownModuleError: module 'default::foo' is not in this schema
```

So the question is what is the right fix here?
- We could ban `::` inside quoted identifiers, for the same reasons we ban `@` and `__` at the beginning of them already. It causes clashes and it's problematic to disambiguate in some circumstances (such as in `__type__.name` string).
- We could review what it means to be a fully-qualified name and whether it really *has* to be 2 identifier tokens separated by a `::` token or whether it's OK to just have a single quoted identifier with the `::` inside of it. This would mean that the type from the example would simply be equivalent to a plain `foo::Bar`, meaning module `foo` type-name `Bar`.

Personally, I'm leaning towards a ban. @1st1 and @elprans , what do you think?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `edb/edgeql/parser/grammar/lexer.py`
Content:
```
1 #
2 # This source file is part of the EdgeDB open source project.
3 #
4 # Copyright 2014-present MagicStack Inc. and the EdgeDB authors.
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 #
10 #     http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #
18 
19 
20 from __future__ import annotations
21 
22 import re
23 
24 from edb.common import lexer
25 
26 from .keywords import edgeql_keywords
27 
28 
29 __all__ = ('EdgeQLLexer',)
30 
31 
32 STATE_KEEP = 0
33 STATE_BASE = 1
34 
35 
36 re_dquote = r'\$([A-Za-z\200-\377_][0-9]*)*\$'
37 
38 Rule = lexer.Rule
39 
40 
41 class UnterminatedStringError(lexer.UnknownTokenError):
42     pass
43 
44 
45 class PseudoRule(Rule):
46     def __init__(self, *, token, regexp, rule_id, next_state=STATE_KEEP):
47         self.id = rule_id
48         Rule._map[rule_id] = self
49         self.token = token
50         self.next_state = next_state
51         self.regexp = regexp
52 
53 
54 class EdgeQLLexer(lexer.Lexer):
55 
56     start_state = STATE_BASE
57 
58     MERGE_TOKENS = {('NAMED', 'ONLY'), ('SET', 'ANNOTATION'), ('SET', 'TYPE')}
59 
60     NL = 'NL'
61     MULTILINE_TOKENS = frozenset(('SCONST', 'BCONST', 'RSCONST'))
62     RE_FLAGS = re.X | re.M | re.I
63 
64     # Basic keywords
65     keyword_rules = [Rule(token=tok[0],
66                           next_state=STATE_KEEP,
67                           regexp=lexer.group(val))
68                      for val, tok in edgeql_keywords.items()]
69 
70     common_rules = keyword_rules + [
71         Rule(token='WS',
72              next_state=STATE_KEEP,
73              regexp=r'[^\S\n]+'),
74 
75         Rule(token='NL',
76              next_state=STATE_KEEP,
77              regexp=r'\n'),
78 
79         Rule(token='COMMENT',
80              next_state=STATE_KEEP,
81              regexp=r'''\#.*?$'''),
82 
83         Rule(token='ASSIGN',
84              next_state=STATE_KEEP,
85              regexp=r':='),
86 
87         Rule(token='REMASSIGN',
88              next_state=STATE_KEEP,
89              regexp=r'-='),
90 
91         Rule(token='ADDASSIGN',
92              next_state=STATE_KEEP,
93              regexp=r'\+='),
94 
95         Rule(token='ARROW',
96              next_state=STATE_KEEP,
97              regexp=r'->'),
98 
99         Rule(token='??',
100              next_state=STATE_KEEP,
101              regexp=r'\?\?'),
102 
103         Rule(token='::',
104              next_state=STATE_KEEP,
105              regexp=r'::'),
106 
107         # special path operators
108         Rule(token='.<',
109              next_state=STATE_KEEP,
110              regexp=r'\.<'),
111 
112         Rule(token='.>',
113              next_state=STATE_KEEP,
114              regexp=r'\.>'),
115 
116         Rule(token='//',
117              next_state=STATE_KEEP,
118              regexp=r'//'),
119 
120         Rule(token='++',
121              next_state=STATE_KEEP,
122              regexp=r'\+\+'),
123 
124         Rule(token='OP',
125              next_state=STATE_KEEP,
126              regexp=r'''
127                 (?: >= | <= | != | \?= | \?!=)
128              '''),
129 
130         Rule(token='self',
131              next_state=STATE_KEEP,
132              regexp=r'[,()\[\].@;:+\-*/%^<>=&|]'),
133 
134         Rule(token='NCONST',
135              next_state=STATE_KEEP,
136              regexp=r"""
137                 (?:
138                     (?: \d+ (?:\.\d+)?
139                         (?:[eE](?:[+\-])?[0-9]+)
140                     )
141                     |
142                     (?: \d+\.\d+)
143                     |
144                     ([1-9]\d* | 0)(?![0-9])
145                 )n
146                 """),
147 
148         Rule(token='FCONST',
149              next_state=STATE_KEEP,
150              regexp=r"""
151                     (?: \d+ (?:\.\d+)?
152                         (?:[eE](?:[+\-])?[0-9]+)
153                     )
154                     |
155                     (?: \d+\.\d+)
156                 """),
157 
158         Rule(token='ICONST',
159              next_state=STATE_KEEP,
160              regexp=r'([1-9]\d* | 0)(?![0-9])'),
161 
162         Rule(token='BCONST',
163              next_state=STATE_KEEP,
164              regexp=rf'''
165                 (?:
166                     b
167                 )
168                 (?P<BQ>
169                     ' | "
170                 )
171                 (?:
172                     (
173                         \\\\ | \\['"] | \n | .
174                         # we'll validate escape codes in the parser
175                     )*?
176                 )
177                 (?P=BQ)
178              '''),
179 
180         Rule(token='RSCONST',
181              next_state=STATE_KEEP,
182              regexp=rf'''
183                 (?:
184                     r
185                 )?
186                 (?P<RQ>
187                     (?:
188                         (?<=r) (?: ' | ")
189                     ) | (?:
190                         (?<!r) (?: {re_dquote})
191                     )
192                 )
193                 (?:
194                     (
195                         \n | .
196                         # we'll validate escape codes in the parser
197                     )*?
198                 )
199                 (?P=RQ)
200              '''),
201 
202         Rule(token='SCONST',
203              next_state=STATE_KEEP,
204              regexp=rf'''
205                 (?P<Q>
206                     ' | "
207                 )
208                 (?:
209                     (
210                         \\\\ | \\['"] | \n | .
211                         # we'll validate escape codes in the parser
212                     )*?
213                 )
214                 (?P=Q)
215              '''),
216 
217         # this rule will capture malformed strings and allow us to
218         # provide better error messages
219         Rule(token='BADSCONST',
220              next_state=STATE_KEEP,
221              regexp=rf'''
222                 [rb]?
223                 (['"] | (?: {re_dquote}))
224                 [^\n]*
225              '''),
226 
227         Rule(token='BADIDENT',
228              next_state=STATE_KEEP,
229              regexp=r'''
230                     __[^\W\d]\w*__
231                     |
232                     `__.*?__`
233                 '''),
234 
235         Rule(token='IDENT',
236              next_state=STATE_KEEP,
237              regexp=r'[^\W\d]\w*'),
238 
239         Rule(token='QIDENT',
240              next_state=STATE_KEEP,
241              regexp=r'`([^`]|``)*`'),
242 
243         Rule(token='self',
244              next_state=STATE_KEEP,
245              regexp=r'[\{\}$]'),
246     ]
247 
248     states = {
249         STATE_BASE:
250             common_rules,
251     }
252 
253     # add capacity to handle a few tokens composed of 2 elements
254     _possible_long_token = {x[0] for x in MERGE_TOKENS}
255     _long_token_match = {x[1]: x[0] for x in MERGE_TOKENS}
256 
257     special_rules = [
258         PseudoRule(token='UNKNOWN',
259                    next_state=STATE_KEEP,
260                    regexp=r'.',
261                    rule_id='err')
262     ]
263 
264     def __init__(self, *, strip_whitespace=True, raise_lexerror=True):
265         super().__init__()
266         self.strip_whitespace = strip_whitespace
267         self.raise_lexerror = raise_lexerror
268 
269     def get_eof_token(self):
270         """Return an EOF token or None if no EOF token is wanted."""
271         return self.token_from_text('EOF', '')
272 
273     def token_from_text(self, rule_token, txt):
274         if rule_token == 'BADSCONST':
275             self.handle_error(f"Unterminated string {txt}",
276                               exact_message=True,
277                               exc_type=UnterminatedStringError)
278         elif rule_token == 'BADIDENT':
279             self.handle_error(txt)
280 
281         elif rule_token == 'QIDENT' and txt[1] == '@':
282             self.handle_error(f'Identifiers cannot start with "@"',
283                               exact_message=True)
284 
285         tok = super().token_from_text(rule_token, txt)
286 
287         if rule_token == 'self':
288             tok = tok._replace(type=txt)
289 
290         elif rule_token == 'QIDENT':
291             # Drop the quotes and replace the "``" inside with a "`"
292             val = txt[1:-1].replace('``', '`')
293             tok = tok._replace(type='IDENT', value=val)
294 
295         return tok
296 
297     def lex(self):
298         buffer = []
299 
300         for tok in super().lex():
301             tok_type = tok.type
302 
303             if self.strip_whitespace and tok_type in {'WS', 'NL', 'COMMENT'}:
304                 # Strip out whitespace and comments
305                 continue
306 
307             elif tok_type in self._possible_long_token:
308                 # Buffer in case this is a merged token
309                 if not buffer:
310                     buffer.append(tok)
311                 else:
312                     yield from iter(buffer)
313                     buffer[:] = [tok]
314 
315             elif tok_type in self._long_token_match:
316                 prev_token = buffer[-1] if buffer else None
317                 if (prev_token and
318                         prev_token.type == self._long_token_match[tok_type]):
319                     tok = prev_token._replace(
320                         value=prev_token.value + ' ' + tok.value,
321                         type=prev_token.type + tok_type)
322                     buffer.pop()
323                 yield tok
324 
325             else:
326                 if buffer:
327                     yield from iter(buffer)
328                     buffer[:] = []
329                 yield tok
330 
331     def lex_highlight(self):
332         return super().lex()
333 
334     def handle_error(self, txt, *,
335                      exact_message=False, exc_type=lexer.UnknownTokenError):
336         if self.raise_lexerror:
337             super().handle_error(
338                 txt, exact_message=exact_message, exc_type=exc_type)
339
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/edb/edgeql/parser/grammar/lexer.py b/edb/edgeql/parser/grammar/lexer.py
--- a/edb/edgeql/parser/grammar/lexer.py
+++ b/edb/edgeql/parser/grammar/lexer.py
@@ -278,9 +278,13 @@
         elif rule_token == 'BADIDENT':
             self.handle_error(txt)
 
-        elif rule_token == 'QIDENT' and txt[1] == '@':
-            self.handle_error(f'Identifiers cannot start with "@"',
-                              exact_message=True)
+        elif rule_token == 'QIDENT':
+            if txt[1] == '@':
+                self.handle_error(f'Identifiers cannot start with "@"',
+                                  exact_message=True)
+            elif '::' in txt:
+                self.handle_error(f'Identifiers cannot contain "::"',
+                                  exact_message=True)
 
         tok = super().token_from_text(rule_token, txt)

{"golden_diff": "diff --git a/edb/edgeql/parser/grammar/lexer.py b/edb/edgeql/parser/grammar/lexer.py\n--- a/edb/edgeql/parser/grammar/lexer.py\n+++ b/edb/edgeql/parser/grammar/lexer.py\n@@ -278,9 +278,13 @@\n         elif rule_token == 'BADIDENT':\n             self.handle_error(txt)\n \n-        elif rule_token == 'QIDENT' and txt[1] == '@':\n-            self.handle_error(f'Identifiers cannot start with \"@\"',\n-                              exact_message=True)\n+        elif rule_token == 'QIDENT':\n+            if txt[1] == '@':\n+                self.handle_error(f'Identifiers cannot start with \"@\"',\n+                                  exact_message=True)\n+            elif '::' in txt:\n+                self.handle_error(f'Identifiers cannot contain \"::\"',\n+                                  exact_message=True)\n \n         tok = super().token_from_text(rule_token, txt)\n", "issue": "Identifier quoting clashes with fully-qualified names\nThe point of identifier quoting is that it allows using non-alphanumeric characters or reserved keywords as identifiers. So consider the following:\r\n```\r\nCREATE TYPE `foo::Bar`;\r\n```\r\nIt will be interpreted as a single identifier, which then gets prefixed with the `default` module name to create a fully-qualified name with module `default` and name `foo::Bar`. Unfortunately, during processing it gets normalized into a single string `\"default::foo::Bar\"`, which then further down the line gets (mis-)interpreted as module `default::foo` and name `Bar`.\r\n```\r\nedgedb> CREATE TYPE `foo::Bar`;\r\nCREATE\r\nedgedb> WITH MODULE schema\r\n....... SELECT ObjectType {name};\r\nUnknownModuleError: module 'default::foo' is not in this schema\r\n```\r\n\r\nSo the question is what is the right fix here?\r\n- We could ban `::` inside quoted identifiers, for the same reasons we ban `@` and `__` at the beginning of them already. It causes clashes and it's problematic to disambiguate in some circumstances (such as in `__type__.name` string).\r\n- We could review what it means to be a fully-qualified name and whether it really *has* to be 2 identifier tokens separated by a `::` token or whether it's OK to just have a single quoted identifier with the `::` inside of it. This would mean that the type from the example would simply be equivalent to a plain `foo::Bar`, meaning module `foo` type-name `Bar`.\r\n\r\nPersonally, I'm leaning towards a ban. @1st1 and @elprans , what do you think?\n", "before_files": [{"content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2014-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport re\n\nfrom edb.common import lexer\n\nfrom .keywords import edgeql_keywords\n\n\n__all__ = ('EdgeQLLexer',)\n\n\nSTATE_KEEP = 0\nSTATE_BASE = 1\n\n\nre_dquote = r'\\$([A-Za-z\\200-\\377_][0-9]*)*\\$'\n\nRule = lexer.Rule\n\n\nclass UnterminatedStringError(lexer.UnknownTokenError):\n    pass\n\n\nclass PseudoRule(Rule):\n    def __init__(self, *, token, regexp, rule_id, next_state=STATE_KEEP):\n        self.id = rule_id\n        Rule._map[rule_id] = self\n        self.token = token\n        self.next_state = next_state\n        self.regexp = regexp\n\n\nclass EdgeQLLexer(lexer.Lexer):\n\n    start_state = STATE_BASE\n\n    MERGE_TOKENS = {('NAMED', 'ONLY'), ('SET', 'ANNOTATION'), ('SET', 'TYPE')}\n\n    NL = 'NL'\n    MULTILINE_TOKENS = frozenset(('SCONST', 'BCONST', 'RSCONST'))\n    RE_FLAGS = re.X | re.M | re.I\n\n    # Basic keywords\n    keyword_rules = [Rule(token=tok[0],\n                          next_state=STATE_KEEP,\n                          regexp=lexer.group(val))\n                     for val, tok in edgeql_keywords.items()]\n\n    common_rules = keyword_rules + [\n        Rule(token='WS',\n             next_state=STATE_KEEP,\n             regexp=r'[^\\S\\n]+'),\n\n        Rule(token='NL',\n             next_state=STATE_KEEP,\n             regexp=r'\\n'),\n\n        Rule(token='COMMENT',\n             next_state=STATE_KEEP,\n             regexp=r'''\\#.*?$'''),\n\n        Rule(token='ASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r':='),\n\n        Rule(token='REMASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r'-='),\n\n        Rule(token='ADDASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r'\\+='),\n\n        Rule(token='ARROW',\n             next_state=STATE_KEEP,\n             regexp=r'->'),\n\n        Rule(token='??',\n             next_state=STATE_KEEP,\n             regexp=r'\\?\\?'),\n\n        Rule(token='::',\n             next_state=STATE_KEEP,\n             regexp=r'::'),\n\n        # special path operators\n        Rule(token='.<',\n             next_state=STATE_KEEP,\n             regexp=r'\\.<'),\n\n        Rule(token='.>',\n             next_state=STATE_KEEP,\n             regexp=r'\\.>'),\n\n        Rule(token='//',\n             next_state=STATE_KEEP,\n             regexp=r'//'),\n\n        Rule(token='++',\n             next_state=STATE_KEEP,\n             regexp=r'\\+\\+'),\n\n        Rule(token='OP',\n             next_state=STATE_KEEP,\n             regexp=r'''\n                (?: >= | <= | != | \\?= | \\?!=)\n             '''),\n\n        Rule(token='self',\n             next_state=STATE_KEEP,\n             regexp=r'[,()\\[\\].@;:+\\-*/%^<>=&|]'),\n\n        Rule(token='NCONST',\n             next_state=STATE_KEEP,\n             regexp=r\"\"\"\n                (?:\n                    (?: \\d+ (?:\\.\\d+)?\n                        (?:[eE](?:[+\\-])?[0-9]+)\n                    )\n                    |\n                    (?: \\d+\\.\\d+)\n                    |\n                    ([1-9]\\d* | 0)(?![0-9])\n                )n\n                \"\"\"),\n\n        Rule(token='FCONST',\n             next_state=STATE_KEEP,\n             regexp=r\"\"\"\n                    (?: \\d+ (?:\\.\\d+)?\n                        (?:[eE](?:[+\\-])?[0-9]+)\n                    )\n                    |\n                    (?: \\d+\\.\\d+)\n                \"\"\"),\n\n        Rule(token='ICONST',\n             next_state=STATE_KEEP,\n             regexp=r'([1-9]\\d* | 0)(?![0-9])'),\n\n        Rule(token='BCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?:\n                    b\n                )\n                (?P<BQ>\n                    ' | \"\n                )\n                (?:\n                    (\n                        \\\\\\\\ | \\\\['\"] | \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=BQ)\n             '''),\n\n        Rule(token='RSCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?:\n                    r\n                )?\n                (?P<RQ>\n                    (?:\n                        (?<=r) (?: ' | \")\n                    ) | (?:\n                        (?<!r) (?: {re_dquote})\n                    )\n                )\n                (?:\n                    (\n                        \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=RQ)\n             '''),\n\n        Rule(token='SCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?P<Q>\n                    ' | \"\n                )\n                (?:\n                    (\n                        \\\\\\\\ | \\\\['\"] | \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=Q)\n             '''),\n\n        # this rule will capture malformed strings and allow us to\n        # provide better error messages\n        Rule(token='BADSCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                [rb]?\n                (['\"] | (?: {re_dquote}))\n                [^\\n]*\n             '''),\n\n        Rule(token='BADIDENT',\n             next_state=STATE_KEEP,\n             regexp=r'''\n                    __[^\\W\\d]\\w*__\n                    |\n                    `__.*?__`\n                '''),\n\n        Rule(token='IDENT',\n             next_state=STATE_KEEP,\n             regexp=r'[^\\W\\d]\\w*'),\n\n        Rule(token='QIDENT',\n             next_state=STATE_KEEP,\n             regexp=r'`([^`]|``)*`'),\n\n        Rule(token='self',\n             next_state=STATE_KEEP,\n             regexp=r'[\\{\\}$]'),\n    ]\n\n    states = {\n        STATE_BASE:\n            common_rules,\n    }\n\n    # add capacity to handle a few tokens composed of 2 elements\n    _possible_long_token = {x[0] for x in MERGE_TOKENS}\n    _long_token_match = {x[1]: x[0] for x in MERGE_TOKENS}\n\n    special_rules = [\n        PseudoRule(token='UNKNOWN',\n                   next_state=STATE_KEEP,\n                   regexp=r'.',\n                   rule_id='err')\n    ]\n\n    def __init__(self, *, strip_whitespace=True, raise_lexerror=True):\n        super().__init__()\n        self.strip_whitespace = strip_whitespace\n        self.raise_lexerror = raise_lexerror\n\n    def get_eof_token(self):\n        \"\"\"Return an EOF token or None if no EOF token is wanted.\"\"\"\n        return self.token_from_text('EOF', '')\n\n    def token_from_text(self, rule_token, txt):\n        if rule_token == 'BADSCONST':\n            self.handle_error(f\"Unterminated string {txt}\",\n                              exact_message=True,\n                              exc_type=UnterminatedStringError)\n        elif rule_token == 'BADIDENT':\n            self.handle_error(txt)\n\n        elif rule_token == 'QIDENT' and txt[1] == '@':\n            self.handle_error(f'Identifiers cannot start with \"@\"',\n                              exact_message=True)\n\n        tok = super().token_from_text(rule_token, txt)\n\n        if rule_token == 'self':\n            tok = tok._replace(type=txt)\n\n        elif rule_token == 'QIDENT':\n            # Drop the quotes and replace the \"``\" inside with a \"`\"\n            val = txt[1:-1].replace('``', '`')\n            tok = tok._replace(type='IDENT', value=val)\n\n        return tok\n\n    def lex(self):\n        buffer = []\n\n        for tok in super().lex():\n            tok_type = tok.type\n\n            if self.strip_whitespace and tok_type in {'WS', 'NL', 'COMMENT'}:\n                # Strip out whitespace and comments\n                continue\n\n            elif tok_type in self._possible_long_token:\n                # Buffer in case this is a merged token\n                if not buffer:\n                    buffer.append(tok)\n                else:\n                    yield from iter(buffer)\n                    buffer[:] = [tok]\n\n            elif tok_type in self._long_token_match:\n                prev_token = buffer[-1] if buffer else None\n                if (prev_token and\n                        prev_token.type == self._long_token_match[tok_type]):\n                    tok = prev_token._replace(\n                        value=prev_token.value + ' ' + tok.value,\n                        type=prev_token.type + tok_type)\n                    buffer.pop()\n                yield tok\n\n            else:\n                if buffer:\n                    yield from iter(buffer)\n                    buffer[:] = []\n                yield tok\n\n    def lex_highlight(self):\n        return super().lex()\n\n    def handle_error(self, txt, *,\n                     exact_message=False, exc_type=lexer.UnknownTokenError):\n        if self.raise_lexerror:\n            super().handle_error(\n                txt, exact_message=exact_message, exc_type=exc_type)\n", "path": "edb/edgeql/parser/grammar/lexer.py"}], "after_files": [{"content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2014-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport re\n\nfrom edb.common import lexer\n\nfrom .keywords import edgeql_keywords\n\n\n__all__ = ('EdgeQLLexer',)\n\n\nSTATE_KEEP = 0\nSTATE_BASE = 1\n\n\nre_dquote = r'\\$([A-Za-z\\200-\\377_][0-9]*)*\\$'\n\nRule = lexer.Rule\n\n\nclass UnterminatedStringError(lexer.UnknownTokenError):\n    pass\n\n\nclass PseudoRule(Rule):\n    def __init__(self, *, token, regexp, rule_id, next_state=STATE_KEEP):\n        self.id = rule_id\n        Rule._map[rule_id] = self\n        self.token = token\n        self.next_state = next_state\n        self.regexp = regexp\n\n\nclass EdgeQLLexer(lexer.Lexer):\n\n    start_state = STATE_BASE\n\n    MERGE_TOKENS = {('NAMED', 'ONLY'), ('SET', 'ANNOTATION'), ('SET', 'TYPE')}\n\n    NL = 'NL'\n    MULTILINE_TOKENS = frozenset(('SCONST', 'BCONST', 'RSCONST'))\n    RE_FLAGS = re.X | re.M | re.I\n\n    # Basic keywords\n    keyword_rules = [Rule(token=tok[0],\n                          next_state=STATE_KEEP,\n                          regexp=lexer.group(val))\n                     for val, tok in edgeql_keywords.items()]\n\n    common_rules = keyword_rules + [\n        Rule(token='WS',\n             next_state=STATE_KEEP,\n             regexp=r'[^\\S\\n]+'),\n\n        Rule(token='NL',\n             next_state=STATE_KEEP,\n             regexp=r'\\n'),\n\n        Rule(token='COMMENT',\n             next_state=STATE_KEEP,\n             regexp=r'''\\#.*?$'''),\n\n        Rule(token='ASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r':='),\n\n        Rule(token='REMASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r'-='),\n\n        Rule(token='ADDASSIGN',\n             next_state=STATE_KEEP,\n             regexp=r'\\+='),\n\n        Rule(token='ARROW',\n             next_state=STATE_KEEP,\n             regexp=r'->'),\n\n        Rule(token='??',\n             next_state=STATE_KEEP,\n             regexp=r'\\?\\?'),\n\n        Rule(token='::',\n             next_state=STATE_KEEP,\n             regexp=r'::'),\n\n        # special path operators\n        Rule(token='.<',\n             next_state=STATE_KEEP,\n             regexp=r'\\.<'),\n\n        Rule(token='.>',\n             next_state=STATE_KEEP,\n             regexp=r'\\.>'),\n\n        Rule(token='//',\n             next_state=STATE_KEEP,\n             regexp=r'//'),\n\n        Rule(token='++',\n             next_state=STATE_KEEP,\n             regexp=r'\\+\\+'),\n\n        Rule(token='OP',\n             next_state=STATE_KEEP,\n             regexp=r'''\n                (?: >= | <= | != | \\?= | \\?!=)\n             '''),\n\n        Rule(token='self',\n             next_state=STATE_KEEP,\n             regexp=r'[,()\\[\\].@;:+\\-*/%^<>=&|]'),\n\n        Rule(token='NCONST',\n             next_state=STATE_KEEP,\n             regexp=r\"\"\"\n                (?:\n                    (?: \\d+ (?:\\.\\d+)?\n                        (?:[eE](?:[+\\-])?[0-9]+)\n                    )\n                    |\n                    (?: \\d+\\.\\d+)\n                    |\n                    ([1-9]\\d* | 0)(?![0-9])\n                )n\n                \"\"\"),\n\n        Rule(token='FCONST',\n             next_state=STATE_KEEP,\n             regexp=r\"\"\"\n                    (?: \\d+ (?:\\.\\d+)?\n                        (?:[eE](?:[+\\-])?[0-9]+)\n                    )\n                    |\n                    (?: \\d+\\.\\d+)\n                \"\"\"),\n\n        Rule(token='ICONST',\n             next_state=STATE_KEEP,\n             regexp=r'([1-9]\\d* | 0)(?![0-9])'),\n\n        Rule(token='BCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?:\n                    b\n                )\n                (?P<BQ>\n                    ' | \"\n                )\n                (?:\n                    (\n                        \\\\\\\\ | \\\\['\"] | \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=BQ)\n             '''),\n\n        Rule(token='RSCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?:\n                    r\n                )?\n                (?P<RQ>\n                    (?:\n                        (?<=r) (?: ' | \")\n                    ) | (?:\n                        (?<!r) (?: {re_dquote})\n                    )\n                )\n                (?:\n                    (\n                        \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=RQ)\n             '''),\n\n        Rule(token='SCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                (?P<Q>\n                    ' | \"\n                )\n                (?:\n                    (\n                        \\\\\\\\ | \\\\['\"] | \\n | .\n                        # we'll validate escape codes in the parser\n                    )*?\n                )\n                (?P=Q)\n             '''),\n\n        # this rule will capture malformed strings and allow us to\n        # provide better error messages\n        Rule(token='BADSCONST',\n             next_state=STATE_KEEP,\n             regexp=rf'''\n                [rb]?\n                (['\"] | (?: {re_dquote}))\n                [^\\n]*\n             '''),\n\n        Rule(token='BADIDENT',\n             next_state=STATE_KEEP,\n             regexp=r'''\n                    __[^\\W\\d]\\w*__\n                    |\n                    `__.*?__`\n                '''),\n\n        Rule(token='IDENT',\n             next_state=STATE_KEEP,\n             regexp=r'[^\\W\\d]\\w*'),\n\n        Rule(token='QIDENT',\n             next_state=STATE_KEEP,\n             regexp=r'`([^`]|``)*`'),\n\n        Rule(token='self',\n             next_state=STATE_KEEP,\n             regexp=r'[\\{\\}$]'),\n    ]\n\n    states = {\n        STATE_BASE:\n            common_rules,\n    }\n\n    # add capacity to handle a few tokens composed of 2 elements\n    _possible_long_token = {x[0] for x in MERGE_TOKENS}\n    _long_token_match = {x[1]: x[0] for x in MERGE_TOKENS}\n\n    special_rules = [\n        PseudoRule(token='UNKNOWN',\n                   next_state=STATE_KEEP,\n                   regexp=r'.',\n                   rule_id='err')\n    ]\n\n    def __init__(self, *, strip_whitespace=True, raise_lexerror=True):\n        super().__init__()\n        self.strip_whitespace = strip_whitespace\n        self.raise_lexerror = raise_lexerror\n\n    def get_eof_token(self):\n        \"\"\"Return an EOF token or None if no EOF token is wanted.\"\"\"\n        return self.token_from_text('EOF', '')\n\n    def token_from_text(self, rule_token, txt):\n        if rule_token == 'BADSCONST':\n            self.handle_error(f\"Unterminated string {txt}\",\n                              exact_message=True,\n                              exc_type=UnterminatedStringError)\n        elif rule_token == 'BADIDENT':\n            self.handle_error(txt)\n\n        elif rule_token == 'QIDENT':\n            if txt[1] == '@':\n                self.handle_error(f'Identifiers cannot start with \"@\"',\n                                  exact_message=True)\n            elif '::' in txt:\n                self.handle_error(f'Identifiers cannot contain \"::\"',\n                                  exact_message=True)\n\n        tok = super().token_from_text(rule_token, txt)\n\n        if rule_token == 'self':\n            tok = tok._replace(type=txt)\n\n        elif rule_token == 'QIDENT':\n            # Drop the quotes and replace the \"``\" inside with a \"`\"\n            val = txt[1:-1].replace('``', '`')\n            tok = tok._replace(type='IDENT', value=val)\n\n        return tok\n\n    def lex(self):\n        buffer = []\n\n        for tok in super().lex():\n            tok_type = tok.type\n\n            if self.strip_whitespace and tok_type in {'WS', 'NL', 'COMMENT'}:\n                # Strip out whitespace and comments\n                continue\n\n            elif tok_type in self._possible_long_token:\n                # Buffer in case this is a merged token\n                if not buffer:\n                    buffer.append(tok)\n                else:\n                    yield from iter(buffer)\n                    buffer[:] = [tok]\n\n            elif tok_type in self._long_token_match:\n                prev_token = buffer[-1] if buffer else None\n                if (prev_token and\n                        prev_token.type == self._long_token_match[tok_type]):\n                    tok = prev_token._replace(\n                        value=prev_token.value + ' ' + tok.value,\n                        type=prev_token.type + tok_type)\n                    buffer.pop()\n                yield tok\n\n            else:\n                if buffer:\n                    yield from iter(buffer)\n                    buffer[:] = []\n                yield tok\n\n    def lex_highlight(self):\n        return super().lex()\n\n    def handle_error(self, txt, *,\n                     exact_message=False, exc_type=lexer.UnknownTokenError):\n        if self.raise_lexerror:\n            super().handle_error(\n                txt, exact_message=exact_message, exc_type=exc_type)\n", "path": "edb/edgeql/parser/grammar/lexer.py"}]}