We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`similar` field in name_variants.yaml should be symmetric and transitive
This comes up in #615.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bin/anthology/index.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 
17 import logging as log
18 import re
19 import yaml
20 from collections import defaultdict, Counter
21 from slugify import slugify
22 from stop_words import get_stop_words
23 from .formatter import bibtex_encode
24 from .people import PersonName
25 from .venues import VenueIndex
26 
27 from typing import List
28 
29 try:
30     from yaml import CLoader as Loader
31 except ImportError:
32     from yaml import Loader
33 
34 
35 BIBKEY_MAX_NAMES = 2
36 
37 
38 def load_stopwords(language):
39     return [t for w in get_stop_words(language) for t in slugify(w).split("-")]
40 
41 
42 class AnthologyIndex:
43     """Keeps an index of persons, their associated papers, paper bibliography
44     keys, etc.."""
45 
46     def __init__(self, parent, srcdir=None):
47         self._parent = parent
48         self.bibkeys = set()
49         self.stopwords = load_stopwords("en")
50         self.id_to_canonical = {}  # maps ids to canonical names
51         self.id_to_used = defaultdict(set)  # maps ids to all names actually used
52         self.name_to_ids = defaultdict(list)  # maps canonical/variant names to ids
53         self.coauthors = defaultdict(Counter)  # maps ids to co-author ids
54         self.comments = {}  # maps ids to comments (used for distinguishing authors with same name)
55         self.similar = defaultdict(set)
56         self.id_to_papers = defaultdict(lambda: defaultdict(list))  # id -> role -> papers
57         self.name_to_papers = defaultdict(lambda: defaultdict(list))  # name -> (explicit id?) -> papers; used only for error checking
58         if srcdir is not None:
59             self.load_variant_list(srcdir)
60 
61     def load_variant_list(self, directory):
62         with open("{}/yaml/name_variants.yaml".format(directory), "r") as f:
63             name_list = yaml.load(f, Loader=Loader)
64 
65             # Reserve ids for people with explicit ids in variant list
66             for entry in name_list:
67                 if "id" in entry:
68                     id_ = entry["id"]
69                     canonical = entry["canonical"]
70                     canonical = PersonName.from_dict(canonical)
71                     self.set_canonical_name(id_, canonical)
72             # Automatically add people with same canonical name to similar list
73             for name, ids in self.name_to_ids.items():
74                 if len(ids) > 1:
75                     for id1 in ids:
76                         for id2 in ids:
77                             if id2 != id1:
78                                 self.similar[id1].add(id2)
79             for entry in name_list:
80                 try:
81                     canonical = entry["canonical"]
82                     variants = entry.get("variants", [])
83                     id_ = entry.get("id", None)
84                 except (KeyError, TypeError):
85                     log.error("Couldn't parse name variant entry: {}".format(entry))
86                     continue
87                 canonical = PersonName.from_dict(canonical)
88                 if id_ is None:
89                     if canonical in self.name_to_ids:
90                         log.error(
91                             "Canonical name '{}' is ambiguous but doesn't have an id; please add one". format(canonical))
92                     id_ = self.fresh_id(canonical)
93                     self.set_canonical_name(id_, canonical)
94                 for variant in variants:
95                     variant = PersonName.from_dict(variant)
96                     if variant in self.name_to_ids:
97                         log.error(
98                             "Tried to add '{}' as variant of '{}', but is already a variant of '{}'".format(
99                                 repr(variant),
100                                 repr(canonical),
101                                 repr(self.id_to_canonical[
102                                     self.name_to_ids[variant][0]
103                                 ]),
104                             )
105                         )
106                         continue
107                     self.add_variant_name(id_, variant)
108                 if "comment" in entry:
109                     self.comments[id_] = entry["comment"]
110                 if "similar" in entry:
111                     self.similar[id_].update(entry["similar"])
112 
113     def _is_stopword(self, word, paper):
114         """Determines if a given word should be considered a stopword for
115         the purpose of generating BibTeX keys."""
116         if word in self.stopwords:
117             return True
118         if paper.is_volume:
119             # Some simple heuristics to exclude probably uninformative words
120             # -- these are not perfect
121             if word in (
122                 "proceedings",
123                 "volume",
124                 "conference",
125                 "workshop",
126                 "annual",
127                 "meeting",
128                 "computational",
129             ):
130                 return True
131             elif (
132                 re.match(r"[0-9]+(st|nd|rd|th)", word)
133                 or word.endswith("ieth")
134                 or word.endswith("enth")
135                 or word
136                 in (
137                     "first",
138                     "second",
139                     "third",
140                     "fourth",
141                     "fifth",
142                     "sixth",
143                     "eighth",
144                     "ninth",
145                     "twelfth",
146                 )
147             ):
148                 return True
149         return False
150 
151     def create_bibkey(self, paper):
152         """Create a unique bibliography key for the given paper."""
153         if paper.is_volume:
154             # Proceedings volumes use venue acronym instead of authors/editors
155             bibnames = slugify(self._parent.venues.get_by_letter(paper.full_id[0]))
156         else:
157             # Regular papers use author/editor names
158             names = paper.get("author")
159             if not names:
160                 names = paper.get("editor", [])
161             if names:
162                 if len(names) > BIBKEY_MAX_NAMES:
163                     bibnames = "{}-etal".format(slugify(names[0][0].last))
164                 else:
165                     bibnames = "-".join(slugify(n.last) for n, _ in names)
166             else:
167                 bibnames = "nn"
168         title = [
169             w
170             for w in slugify(paper.get_title("plain")).split("-")
171             if not self._is_stopword(w, paper)
172         ]
173         bibkey = "{}-{}-{}".format(bibnames, str(paper.get("year")), title.pop(0))
174         while bibkey in self.bibkeys:  # guarantee uniqueness
175             if title:
176                 bibkey += "-{}".format(title.pop(0))
177             else:
178                 match = re.search(r"-([0-9][0-9]?)$", bibkey)
179                 if match is not None:
180                     num = int(match.group(1)) + 1
181                     bibkey = bibkey[: -len(match.group(1))] + "{}".format(num)
182                 else:
183                     bibkey += "-2"
184                 log.debug(
185                     "New bibkey for clash that can't be resolved by adding title words: {}".format(
186                         bibkey
187                     )
188                 )
189         self.bibkeys.add(bibkey)
190         return bibkey
191 
192     def register(self, paper):
193         """Register all names associated with the given paper."""
194         from .papers import Paper
195 
196         assert isinstance(paper, Paper), "Expected Paper, got {} ({})".format(
197             type(paper), repr(paper)
198         )
199         paper.bibkey = self.create_bibkey(paper)
200         for role in ("author", "editor"):
201             for name, id_ in paper.get(role, []):
202                 if id_ is None:
203                     if len(self.name_to_ids.get(name, [])) > 1:
204                         log.error("Paper {} uses ambiguous name '{}' without id".format(paper.full_id, name))
205                         log.error("  Please add an id, for example: {}".format(" ".join(self.name_to_ids[name])))
206                     id_ = self.resolve_name(name)["id"]
207                     explicit = False
208                 else:
209                     if id_ not in self.id_to_canonical:
210                         log.error("Paper {} uses name '{}' with id '{}' that does not exist".format(paper.full_id, name, id_))
211                     explicit = True
212                     
213                 self.id_to_used[id_].add(name)
214                 # Register paper
215                 self.id_to_papers[id_][role].append(paper.full_id)
216                 self.name_to_papers[name][explicit].append(paper.full_id)
217                 # Register co-author(s)
218                 for co_name, co_id in paper.get(role):
219                     if co_id is None:
220                         co_id = self.resolve_name(co_name)["id"]
221                     if co_id != id_:
222                         self.coauthors[id_][co_id] += 1
223 
224     def verify(self):
225         for name, ids in self.name_to_ids.items():
226             for id_ in ids:
227                 cname = self.id_to_canonical[id_]
228                 if name != cname and name not in self.id_to_used[id_]:
229                     log.warning(
230                         "Variant name '{}' of '{}' is not used".format(
231                             repr(name),
232                             repr(cname)))
233         for name, d in self.name_to_papers.items():
234             if len(d[False]) > 0 and len(d[True]) > 0:
235                 log.error("Name '{}' is used both with and without explicit id".format(repr(name)))
236                 log.error(
237                     "  Please add an id to paper(s):   {}".format(
238                         " ".join(d[False])
239                     )
240                 )
241                 log.error(
242                     "  Or remove the id from paper(s): {}".format(
243                         " ".join(d[True])
244                     )
245                 )
246         
247     def personids(self):
248         return self.id_to_canonical.keys()
249 
250     def get_canonical_name(self, id_):
251         return self.id_to_canonical[id_]
252 
253     def set_canonical_name(self, id_, name):
254         if id_ in self.id_to_canonical:
255             log.error("Person id '{}' is used by both '{}' and '{}'".format(id_, name, self.id_to_canonical[id_]))
256         self.id_to_canonical[id_] = name
257         self.name_to_ids[name].append(id_)
258 
259     def add_variant_name(self, id_, name):
260         self.name_to_ids[name].append(id_)
261 
262     def get_used_names(self, id_):
263         """Return a list of all names used for a given person."""
264         return self.id_to_used[id_]
265 
266     def get_ids(self, name: PersonName) -> List[str]:
267         """
268         Returns a list of distinct IDs (people) associated with a surface form.
269 
270         :param name: The name (surface form) of the person being searched (id field ignored).
271         :return: A list of name ID strings.
272         """
273         if name not in self.name_to_ids:
274             id_ = self.fresh_id(name)
275             self.set_canonical_name(id_, name)
276 
277         return sorted(self.name_to_ids[name])
278 
279     def resolve_name(self, name, id_=None):
280         """Find person named 'name' and return a dict with fields
281         'first', 'last', 'id'"""
282         if id_ is None:
283             ids = self.get_ids(name)
284             assert len(ids) > 0
285             if len(ids) > 1:
286                 log.debug("Name '{}' is ambiguous between {}".format(
287                     repr(name),
288                     ', '.join("'{}'".format(i) for i in ids)
289                 ))
290             # Just return the first
291             id_ = ids[0]
292         d = name.as_dict()
293         d["id"] = id_
294         return d
295 
296     def fresh_id(self, name):
297         assert name not in self.name_to_ids, name
298         slug, i = slugify(repr(name)), 0
299         while slug == "" or slug in self.id_to_canonical:
300             i += 1
301             slug = "{}{}".format(slugify(repr(name)), i)
302         return slug
303 
304     def get_papers(self, id_, role=None):
305         if role is None:
306             return [p for p_list in self.id_to_papers[id_].values() for p in p_list]
307         return self.id_to_papers[id_][role]
308 
309     def get_coauthors(self, id_):
310         return self.coauthors[id_].items()
311 
312     def get_venues(self, vidx: VenueIndex, id_):
313         """Get a list of venues a person has published in, with counts."""
314         venues = Counter()
315         for paper in self.get_papers(id_):
316             for venue in vidx.get_associated_venues(paper):
317                 venues[venue] += 1
318         return venues
319
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bin/anthology/index.py b/bin/anthology/index.py
--- a/bin/anthology/index.py
+++ b/bin/anthology/index.py
@@ -109,6 +109,22 @@
                     self.comments[id_] = entry["comment"]
                 if "similar" in entry:
                     self.similar[id_].update(entry["similar"])
+                    for other in entry["similar"]:
+                        if id_ not in self.similar[other]:
+                            log.debug('inferring similar name {} -> {}'.format(other, id_))
+                        self.similar[other].add(id_)
+                        
+        # form transitive closure of self.similar
+        again = True
+        while again:
+            again = False
+            for x in list(self.similar):
+                for y in list(self.similar[x]):
+                    for z in list(self.similar[y]):
+                        if z != x and z not in self.similar[x]:
+                            self.similar[x].add(z)
+                            log.debug('inferring similar name {} -> {}'.format(x, z))
+                            again = True
 
     def _is_stopword(self, word, paper):
         """Determines if a given word should be considered a stopword for

{"golden_diff": "diff --git a/bin/anthology/index.py b/bin/anthology/index.py\n--- a/bin/anthology/index.py\n+++ b/bin/anthology/index.py\n@@ -109,6 +109,22 @@\n                     self.comments[id_] = entry[\"comment\"]\n                 if \"similar\" in entry:\n                     self.similar[id_].update(entry[\"similar\"])\n+                    for other in entry[\"similar\"]:\n+                        if id_ not in self.similar[other]:\n+                            log.debug('inferring similar name {} -> {}'.format(other, id_))\n+                        self.similar[other].add(id_)\n+                        \n+        # form transitive closure of self.similar\n+        again = True\n+        while again:\n+            again = False\n+            for x in list(self.similar):\n+                for y in list(self.similar[x]):\n+                    for z in list(self.similar[y]):\n+                        if z != x and z not in self.similar[x]:\n+                            self.similar[x].add(z)\n+                            log.debug('inferring similar name {} -> {}'.format(x, z))\n+                            again = True\n \n     def _is_stopword(self, word, paper):\n         \"\"\"Determines if a given word should be considered a stopword for\n", "issue": "`similar` field in name_variants.yaml should be symmetric and transitive\nThis comes up in #615.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n#\n# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging as log\nimport re\nimport yaml\nfrom collections import defaultdict, Counter\nfrom slugify import slugify\nfrom stop_words import get_stop_words\nfrom .formatter import bibtex_encode\nfrom .people import PersonName\nfrom .venues import VenueIndex\n\nfrom typing import List\n\ntry:\n    from yaml import CLoader as Loader\nexcept ImportError:\n    from yaml import Loader\n\n\nBIBKEY_MAX_NAMES = 2\n\n\ndef load_stopwords(language):\n    return [t for w in get_stop_words(language) for t in slugify(w).split(\"-\")]\n\n\nclass AnthologyIndex:\n    \"\"\"Keeps an index of persons, their associated papers, paper bibliography\n    keys, etc..\"\"\"\n\n    def __init__(self, parent, srcdir=None):\n        self._parent = parent\n        self.bibkeys = set()\n        self.stopwords = load_stopwords(\"en\")\n        self.id_to_canonical = {}  # maps ids to canonical names\n        self.id_to_used = defaultdict(set)  # maps ids to all names actually used\n        self.name_to_ids = defaultdict(list)  # maps canonical/variant names to ids\n        self.coauthors = defaultdict(Counter)  # maps ids to co-author ids\n        self.comments = {}  # maps ids to comments (used for distinguishing authors with same name)\n        self.similar = defaultdict(set)\n        self.id_to_papers = defaultdict(lambda: defaultdict(list))  # id -> role -> papers\n        self.name_to_papers = defaultdict(lambda: defaultdict(list))  # name -> (explicit id?) -> papers; used only for error checking\n        if srcdir is not None:\n            self.load_variant_list(srcdir)\n\n    def load_variant_list(self, directory):\n        with open(\"{}/yaml/name_variants.yaml\".format(directory), \"r\") as f:\n            name_list = yaml.load(f, Loader=Loader)\n\n            # Reserve ids for people with explicit ids in variant list\n            for entry in name_list:\n                if \"id\" in entry:\n                    id_ = entry[\"id\"]\n                    canonical = entry[\"canonical\"]\n                    canonical = PersonName.from_dict(canonical)\n                    self.set_canonical_name(id_, canonical)\n            # Automatically add people with same canonical name to similar list\n            for name, ids in self.name_to_ids.items():\n                if len(ids) > 1:\n                    for id1 in ids:\n                        for id2 in ids:\n                            if id2 != id1:\n                                self.similar[id1].add(id2)\n            for entry in name_list:\n                try:\n                    canonical = entry[\"canonical\"]\n                    variants = entry.get(\"variants\", [])\n                    id_ = entry.get(\"id\", None)\n                except (KeyError, TypeError):\n                    log.error(\"Couldn't parse name variant entry: {}\".format(entry))\n                    continue\n                canonical = PersonName.from_dict(canonical)\n                if id_ is None:\n                    if canonical in self.name_to_ids:\n                        log.error(\n                            \"Canonical name '{}' is ambiguous but doesn't have an id; please add one\". format(canonical))\n                    id_ = self.fresh_id(canonical)\n                    self.set_canonical_name(id_, canonical)\n                for variant in variants:\n                    variant = PersonName.from_dict(variant)\n                    if variant in self.name_to_ids:\n                        log.error(\n                            \"Tried to add '{}' as variant of '{}', but is already a variant of '{}'\".format(\n                                repr(variant),\n                                repr(canonical),\n                                repr(self.id_to_canonical[\n                                    self.name_to_ids[variant][0]\n                                ]),\n                            )\n                        )\n                        continue\n                    self.add_variant_name(id_, variant)\n                if \"comment\" in entry:\n                    self.comments[id_] = entry[\"comment\"]\n                if \"similar\" in entry:\n                    self.similar[id_].update(entry[\"similar\"])\n\n    def _is_stopword(self, word, paper):\n        \"\"\"Determines if a given word should be considered a stopword for\n        the purpose of generating BibTeX keys.\"\"\"\n        if word in self.stopwords:\n            return True\n        if paper.is_volume:\n            # Some simple heuristics to exclude probably uninformative words\n            # -- these are not perfect\n            if word in (\n                \"proceedings\",\n                \"volume\",\n                \"conference\",\n                \"workshop\",\n                \"annual\",\n                \"meeting\",\n                \"computational\",\n            ):\n                return True\n            elif (\n                re.match(r\"[0-9]+(st|nd|rd|th)\", word)\n                or word.endswith(\"ieth\")\n                or word.endswith(\"enth\")\n                or word\n                in (\n                    \"first\",\n                    \"second\",\n                    \"third\",\n                    \"fourth\",\n                    \"fifth\",\n                    \"sixth\",\n                    \"eighth\",\n                    \"ninth\",\n                    \"twelfth\",\n                )\n            ):\n                return True\n        return False\n\n    def create_bibkey(self, paper):\n        \"\"\"Create a unique bibliography key for the given paper.\"\"\"\n        if paper.is_volume:\n            # Proceedings volumes use venue acronym instead of authors/editors\n            bibnames = slugify(self._parent.venues.get_by_letter(paper.full_id[0]))\n        else:\n            # Regular papers use author/editor names\n            names = paper.get(\"author\")\n            if not names:\n                names = paper.get(\"editor\", [])\n            if names:\n                if len(names) > BIBKEY_MAX_NAMES:\n                    bibnames = \"{}-etal\".format(slugify(names[0][0].last))\n                else:\n                    bibnames = \"-\".join(slugify(n.last) for n, _ in names)\n            else:\n                bibnames = \"nn\"\n        title = [\n            w\n            for w in slugify(paper.get_title(\"plain\")).split(\"-\")\n            if not self._is_stopword(w, paper)\n        ]\n        bibkey = \"{}-{}-{}\".format(bibnames, str(paper.get(\"year\")), title.pop(0))\n        while bibkey in self.bibkeys:  # guarantee uniqueness\n            if title:\n                bibkey += \"-{}\".format(title.pop(0))\n            else:\n                match = re.search(r\"-([0-9][0-9]?)$\", bibkey)\n                if match is not None:\n                    num = int(match.group(1)) + 1\n                    bibkey = bibkey[: -len(match.group(1))] + \"{}\".format(num)\n                else:\n                    bibkey += \"-2\"\n                log.debug(\n                    \"New bibkey for clash that can't be resolved by adding title words: {}\".format(\n                        bibkey\n                    )\n                )\n        self.bibkeys.add(bibkey)\n        return bibkey\n\n    def register(self, paper):\n        \"\"\"Register all names associated with the given paper.\"\"\"\n        from .papers import Paper\n\n        assert isinstance(paper, Paper), \"Expected Paper, got {} ({})\".format(\n            type(paper), repr(paper)\n        )\n        paper.bibkey = self.create_bibkey(paper)\n        for role in (\"author\", \"editor\"):\n            for name, id_ in paper.get(role, []):\n                if id_ is None:\n                    if len(self.name_to_ids.get(name, [])) > 1:\n                        log.error(\"Paper {} uses ambiguous name '{}' without id\".format(paper.full_id, name))\n                        log.error(\"  Please add an id, for example: {}\".format(\" \".join(self.name_to_ids[name])))\n                    id_ = self.resolve_name(name)[\"id\"]\n                    explicit = False\n                else:\n                    if id_ not in self.id_to_canonical:\n                        log.error(\"Paper {} uses name '{}' with id '{}' that does not exist\".format(paper.full_id, name, id_))\n                    explicit = True\n                    \n                self.id_to_used[id_].add(name)\n                # Register paper\n                self.id_to_papers[id_][role].append(paper.full_id)\n                self.name_to_papers[name][explicit].append(paper.full_id)\n                # Register co-author(s)\n                for co_name, co_id in paper.get(role):\n                    if co_id is None:\n                        co_id = self.resolve_name(co_name)[\"id\"]\n                    if co_id != id_:\n                        self.coauthors[id_][co_id] += 1\n\n    def verify(self):\n        for name, ids in self.name_to_ids.items():\n            for id_ in ids:\n                cname = self.id_to_canonical[id_]\n                if name != cname and name not in self.id_to_used[id_]:\n                    log.warning(\n                        \"Variant name '{}' of '{}' is not used\".format(\n                            repr(name),\n                            repr(cname)))\n        for name, d in self.name_to_papers.items():\n            if len(d[False]) > 0 and len(d[True]) > 0:\n                log.error(\"Name '{}' is used both with and without explicit id\".format(repr(name)))\n                log.error(\n                    \"  Please add an id to paper(s):   {}\".format(\n                        \" \".join(d[False])\n                    )\n                )\n                log.error(\n                    \"  Or remove the id from paper(s): {}\".format(\n                        \" \".join(d[True])\n                    )\n                )\n        \n    def personids(self):\n        return self.id_to_canonical.keys()\n\n    def get_canonical_name(self, id_):\n        return self.id_to_canonical[id_]\n\n    def set_canonical_name(self, id_, name):\n        if id_ in self.id_to_canonical:\n            log.error(\"Person id '{}' is used by both '{}' and '{}'\".format(id_, name, self.id_to_canonical[id_]))\n        self.id_to_canonical[id_] = name\n        self.name_to_ids[name].append(id_)\n\n    def add_variant_name(self, id_, name):\n        self.name_to_ids[name].append(id_)\n\n    def get_used_names(self, id_):\n        \"\"\"Return a list of all names used for a given person.\"\"\"\n        return self.id_to_used[id_]\n\n    def get_ids(self, name: PersonName) -> List[str]:\n        \"\"\"\n        Returns a list of distinct IDs (people) associated with a surface form.\n\n        :param name: The name (surface form) of the person being searched (id field ignored).\n        :return: A list of name ID strings.\n        \"\"\"\n        if name not in self.name_to_ids:\n            id_ = self.fresh_id(name)\n            self.set_canonical_name(id_, name)\n\n        return sorted(self.name_to_ids[name])\n\n    def resolve_name(self, name, id_=None):\n        \"\"\"Find person named 'name' and return a dict with fields\n        'first', 'last', 'id'\"\"\"\n        if id_ is None:\n            ids = self.get_ids(name)\n            assert len(ids) > 0\n            if len(ids) > 1:\n                log.debug(\"Name '{}' is ambiguous between {}\".format(\n                    repr(name),\n                    ', '.join(\"'{}'\".format(i) for i in ids)\n                ))\n            # Just return the first\n            id_ = ids[0]\n        d = name.as_dict()\n        d[\"id\"] = id_\n        return d\n\n    def fresh_id(self, name):\n        assert name not in self.name_to_ids, name\n        slug, i = slugify(repr(name)), 0\n        while slug == \"\" or slug in self.id_to_canonical:\n            i += 1\n            slug = \"{}{}\".format(slugify(repr(name)), i)\n        return slug\n\n    def get_papers(self, id_, role=None):\n        if role is None:\n            return [p for p_list in self.id_to_papers[id_].values() for p in p_list]\n        return self.id_to_papers[id_][role]\n\n    def get_coauthors(self, id_):\n        return self.coauthors[id_].items()\n\n    def get_venues(self, vidx: VenueIndex, id_):\n        \"\"\"Get a list of venues a person has published in, with counts.\"\"\"\n        venues = Counter()\n        for paper in self.get_papers(id_):\n            for venue in vidx.get_associated_venues(paper):\n                venues[venue] += 1\n        return venues\n", "path": "bin/anthology/index.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n#\n# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging as log\nimport re\nimport yaml\nfrom collections import defaultdict, Counter\nfrom slugify import slugify\nfrom stop_words import get_stop_words\nfrom .formatter import bibtex_encode\nfrom .people import PersonName\nfrom .venues import VenueIndex\n\nfrom typing import List\n\ntry:\n    from yaml import CLoader as Loader\nexcept ImportError:\n    from yaml import Loader\n\n\nBIBKEY_MAX_NAMES = 2\n\n\ndef load_stopwords(language):\n    return [t for w in get_stop_words(language) for t in slugify(w).split(\"-\")]\n\n\nclass AnthologyIndex:\n    \"\"\"Keeps an index of persons, their associated papers, paper bibliography\n    keys, etc..\"\"\"\n\n    def __init__(self, parent, srcdir=None):\n        self._parent = parent\n        self.bibkeys = set()\n        self.stopwords = load_stopwords(\"en\")\n        self.id_to_canonical = {}  # maps ids to canonical names\n        self.id_to_used = defaultdict(set)  # maps ids to all names actually used\n        self.name_to_ids = defaultdict(list)  # maps canonical/variant names to ids\n        self.coauthors = defaultdict(Counter)  # maps ids to co-author ids\n        self.comments = {}  # maps ids to comments (used for distinguishing authors with same name)\n        self.similar = defaultdict(set)\n        self.id_to_papers = defaultdict(lambda: defaultdict(list))  # id -> role -> papers\n        self.name_to_papers = defaultdict(lambda: defaultdict(list))  # name -> (explicit id?) -> papers; used only for error checking\n        if srcdir is not None:\n            self.load_variant_list(srcdir)\n\n    def load_variant_list(self, directory):\n        with open(\"{}/yaml/name_variants.yaml\".format(directory), \"r\") as f:\n            name_list = yaml.load(f, Loader=Loader)\n\n            # Reserve ids for people with explicit ids in variant list\n            for entry in name_list:\n                if \"id\" in entry:\n                    id_ = entry[\"id\"]\n                    canonical = entry[\"canonical\"]\n                    canonical = PersonName.from_dict(canonical)\n                    self.set_canonical_name(id_, canonical)\n            # Automatically add people with same canonical name to similar list\n            for name, ids in self.name_to_ids.items():\n                if len(ids) > 1:\n                    for id1 in ids:\n                        for id2 in ids:\n                            if id2 != id1:\n                                self.similar[id1].add(id2)\n            for entry in name_list:\n                try:\n                    canonical = entry[\"canonical\"]\n                    variants = entry.get(\"variants\", [])\n                    id_ = entry.get(\"id\", None)\n                except (KeyError, TypeError):\n                    log.error(\"Couldn't parse name variant entry: {}\".format(entry))\n                    continue\n                canonical = PersonName.from_dict(canonical)\n                if id_ is None:\n                    if canonical in self.name_to_ids:\n                        log.error(\n                            \"Canonical name '{}' is ambiguous but doesn't have an id; please add one\". format(canonical))\n                    id_ = self.fresh_id(canonical)\n                    self.set_canonical_name(id_, canonical)\n                for variant in variants:\n                    variant = PersonName.from_dict(variant)\n                    if variant in self.name_to_ids:\n                        log.error(\n                            \"Tried to add '{}' as variant of '{}', but is already a variant of '{}'\".format(\n                                repr(variant),\n                                repr(canonical),\n                                repr(self.id_to_canonical[\n                                    self.name_to_ids[variant][0]\n                                ]),\n                            )\n                        )\n                        continue\n                    self.add_variant_name(id_, variant)\n                if \"comment\" in entry:\n                    self.comments[id_] = entry[\"comment\"]\n                if \"similar\" in entry:\n                    self.similar[id_].update(entry[\"similar\"])\n                    for other in entry[\"similar\"]:\n                        if id_ not in self.similar[other]:\n                            log.debug('inferring similar name {} -> {}'.format(other, id_))\n                        self.similar[other].add(id_)\n                        \n        # form transitive closure of self.similar\n        again = True\n        while again:\n            again = False\n            for x in list(self.similar):\n                for y in list(self.similar[x]):\n                    for z in list(self.similar[y]):\n                        if z != x and z not in self.similar[x]:\n                            self.similar[x].add(z)\n                            log.debug('inferring similar name {} -> {}'.format(x, z))\n                            again = True\n\n    def _is_stopword(self, word, paper):\n        \"\"\"Determines if a given word should be considered a stopword for\n        the purpose of generating BibTeX keys.\"\"\"\n        if word in self.stopwords:\n            return True\n        if paper.is_volume:\n            # Some simple heuristics to exclude probably uninformative words\n            # -- these are not perfect\n            if word in (\n                \"proceedings\",\n                \"volume\",\n                \"conference\",\n                \"workshop\",\n                \"annual\",\n                \"meeting\",\n                \"computational\",\n            ):\n                return True\n            elif (\n                re.match(r\"[0-9]+(st|nd|rd|th)\", word)\n                or word.endswith(\"ieth\")\n                or word.endswith(\"enth\")\n                or word\n                in (\n                    \"first\",\n                    \"second\",\n                    \"third\",\n                    \"fourth\",\n                    \"fifth\",\n                    \"sixth\",\n                    \"eighth\",\n                    \"ninth\",\n                    \"twelfth\",\n                )\n            ):\n                return True\n        return False\n\n    def create_bibkey(self, paper):\n        \"\"\"Create a unique bibliography key for the given paper.\"\"\"\n        if paper.is_volume:\n            # Proceedings volumes use venue acronym instead of authors/editors\n            bibnames = slugify(self._parent.venues.get_by_letter(paper.full_id[0]))\n        else:\n            # Regular papers use author/editor names\n            names = paper.get(\"author\")\n            if not names:\n                names = paper.get(\"editor\", [])\n            if names:\n                if len(names) > BIBKEY_MAX_NAMES:\n                    bibnames = \"{}-etal\".format(slugify(names[0][0].last))\n                else:\n                    bibnames = \"-\".join(slugify(n.last) for n, _ in names)\n            else:\n                bibnames = \"nn\"\n        title = [\n            w\n            for w in slugify(paper.get_title(\"plain\")).split(\"-\")\n            if not self._is_stopword(w, paper)\n        ]\n        bibkey = \"{}-{}-{}\".format(bibnames, str(paper.get(\"year\")), title.pop(0))\n        while bibkey in self.bibkeys:  # guarantee uniqueness\n            if title:\n                bibkey += \"-{}\".format(title.pop(0))\n            else:\n                match = re.search(r\"-([0-9][0-9]?)$\", bibkey)\n                if match is not None:\n                    num = int(match.group(1)) + 1\n                    bibkey = bibkey[: -len(match.group(1))] + \"{}\".format(num)\n                else:\n                    bibkey += \"-2\"\n                log.debug(\n                    \"New bibkey for clash that can't be resolved by adding title words: {}\".format(\n                        bibkey\n                    )\n                )\n        self.bibkeys.add(bibkey)\n        return bibkey\n\n    def register(self, paper):\n        \"\"\"Register all names associated with the given paper.\"\"\"\n        from .papers import Paper\n\n        assert isinstance(paper, Paper), \"Expected Paper, got {} ({})\".format(\n            type(paper), repr(paper)\n        )\n        paper.bibkey = self.create_bibkey(paper)\n        for role in (\"author\", \"editor\"):\n            for name, id_ in paper.get(role, []):\n                if id_ is None:\n                    if len(self.name_to_ids.get(name, [])) > 1:\n                        log.error(\"Paper {} uses ambiguous name '{}' without id\".format(paper.full_id, name))\n                        log.error(\"  Please add an id, for example: {}\".format(\" \".join(self.name_to_ids[name])))\n                    id_ = self.resolve_name(name)[\"id\"]\n                    explicit = False\n                else:\n                    if id_ not in self.id_to_canonical:\n                        log.error(\"Paper {} uses name '{}' with id '{}' that does not exist\".format(paper.full_id, name, id_))\n                    explicit = True\n                    \n                self.id_to_used[id_].add(name)\n                # Register paper\n                self.id_to_papers[id_][role].append(paper.full_id)\n                self.name_to_papers[name][explicit].append(paper.full_id)\n                # Register co-author(s)\n                for co_name, co_id in paper.get(role):\n                    if co_id is None:\n                        co_id = self.resolve_name(co_name)[\"id\"]\n                    if co_id != id_:\n                        self.coauthors[id_][co_id] += 1\n\n    def verify(self):\n        for name, ids in self.name_to_ids.items():\n            for id_ in ids:\n                cname = self.id_to_canonical[id_]\n                if name != cname and name not in self.id_to_used[id_]:\n                    log.warning(\n                        \"Variant name '{}' of '{}' is not used\".format(\n                            repr(name),\n                            repr(cname)))\n        for name, d in self.name_to_papers.items():\n            if len(d[False]) > 0 and len(d[True]) > 0:\n                log.error(\"Name '{}' is used both with and without explicit id\".format(repr(name)))\n                log.error(\n                    \"  Please add an id to paper(s):   {}\".format(\n                        \" \".join(d[False])\n                    )\n                )\n                log.error(\n                    \"  Or remove the id from paper(s): {}\".format(\n                        \" \".join(d[True])\n                    )\n                )\n        \n    def personids(self):\n        return self.id_to_canonical.keys()\n\n    def get_canonical_name(self, id_):\n        return self.id_to_canonical[id_]\n\n    def set_canonical_name(self, id_, name):\n        if id_ in self.id_to_canonical:\n            log.error(\"Person id '{}' is used by both '{}' and '{}'\".format(id_, name, self.id_to_canonical[id_]))\n        self.id_to_canonical[id_] = name\n        self.name_to_ids[name].append(id_)\n\n    def add_variant_name(self, id_, name):\n        self.name_to_ids[name].append(id_)\n\n    def get_used_names(self, id_):\n        \"\"\"Return a list of all names used for a given person.\"\"\"\n        return self.id_to_used[id_]\n\n    def get_ids(self, name: PersonName) -> List[str]:\n        \"\"\"\n        Returns a list of distinct IDs (people) associated with a surface form.\n\n        :param name: The name (surface form) of the person being searched (id field ignored).\n        :return: A list of name ID strings.\n        \"\"\"\n        if name not in self.name_to_ids:\n            id_ = self.fresh_id(name)\n            self.set_canonical_name(id_, name)\n\n        return sorted(self.name_to_ids[name])\n\n    def resolve_name(self, name, id_=None):\n        \"\"\"Find person named 'name' and return a dict with fields\n        'first', 'last', 'id'\"\"\"\n        if id_ is None:\n            ids = self.get_ids(name)\n            assert len(ids) > 0\n            if len(ids) > 1:\n                log.debug(\"Name '{}' is ambiguous between {}\".format(\n                    repr(name),\n                    ', '.join(\"'{}'\".format(i) for i in ids)\n                ))\n            # Just return the first\n            id_ = ids[0]\n        d = name.as_dict()\n        d[\"id\"] = id_\n        return d\n\n    def fresh_id(self, name):\n        assert name not in self.name_to_ids, name\n        slug, i = slugify(repr(name)), 0\n        while slug == \"\" or slug in self.id_to_canonical:\n            i += 1\n            slug = \"{}{}\".format(slugify(repr(name)), i)\n        return slug\n\n    def get_papers(self, id_, role=None):\n        if role is None:\n            return [p for p_list in self.id_to_papers[id_].values() for p in p_list]\n        return self.id_to_papers[id_][role]\n\n    def get_coauthors(self, id_):\n        return self.coauthors[id_].items()\n\n    def get_venues(self, vidx: VenueIndex, id_):\n        \"\"\"Get a list of venues a person has published in, with counts.\"\"\"\n        venues = Counter()\n        for paper in self.get_papers(id_):\n            for venue in vidx.get_associated_venues(paper):\n                venues[venue] += 1\n        return venues\n", "path": "bin/anthology/index.py"}]}