We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
KS failing since at least 2019-02-24
KS has been failing since 2019-02-24

Based on automated runs it appears that KS has not run successfully in 2 days (2019-02-24).


```
  06:02:22 INFO pupa: save jurisdiction Kansas as jurisdiction_ocd-jurisdiction-country:us-state:ks-government.json
06:02:22 INFO pupa: save organization Kansas State Legislature as organization_ea95774a-38c2-11e9-b4ad-021e9c8c6450.json
06:02:22 INFO pupa: save organization Senate as organization_ea957a1a-38c2-11e9-b4ad-021e9c8c6450.json
06:02:22 INFO pupa: save organization House as organization_ea957bbe-38c2-11e9-b4ad-021e9c8c6450.json
06:02:22 INFO pupa: no session specified, using 2019-2020
06:02:22 INFO scrapelib: GET - http://www.kslegislature.org/li/api/v11/rev-1/bill_status/
06:02:23 WARNING pupa: unknown action code on SB9: misc_he_200 Hearing: Tuesday, January 22, 2019, 10:30 AM Room 548-S
06:02:23 WARNING pupa: unknown action code on SB9: misc_he_200 Hearing: Tuesday, February 12, 2019, 9:00 AM Room 112-N
06:02:23 INFO scrapelib: GET - http://www.kslegislature.org/li/b2019_20/measures/sb9/
06:02:27 INFO scrapelib: GET - http://www.kslegislature.org/li/b2019_20/measures/vote_view/je_20190222111337_914158/
06:02:32 WARNING scrapelib: sleeping for 10 seconds before retry
06:02:42 WARNING scrapelib: sleeping for 20 seconds before retry
06:03:03 WARNING scrapelib: sleeping for 40 seconds before retry
```

Visit http://bobsled.openstates.org for more info.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `openstates/ks/bills.py`
Content:
```
1 import re
2 import json
3 import datetime
4 
5 import lxml.html
6 from pupa.scrape import Scraper, Bill, VoteEvent
7 
8 from . import ksapi
9 
10 
11 def _clean_spaces(title):
12     return re.sub(r'\s+', ' ', title)
13 
14 
15 class KSBillScraper(Scraper):
16     def scrape(self, chamber=None, session=None):
17         if session is None:
18             session = self.latest_session()
19             self.info('no session specified, using %s', session)
20 
21         chambers = [chamber] if chamber is not None else ['upper', 'lower']
22 
23         for chamber in chambers:
24             yield from self.scrape_chamber(chamber, session)
25 
26     def scrape_chamber(self, chamber, session):
27         chamber_name = 'Senate' if chamber == 'upper' else 'House'
28         chamber_letter = chamber_name[0]
29         # perhaps we should save this data so we can make one request for both?
30         bill_request = self.get(ksapi.url + 'bill_status/').text
31         bill_request_json = json.loads(bill_request)
32         bills = bill_request_json['content']
33         for bill_data in bills:
34 
35             bill_id = bill_data['BILLNO']
36 
37             # filter other chambers
38             if not bill_id.startswith(chamber_letter):
39                 continue
40 
41             if 'CR' in bill_id:
42                 btype = 'concurrent resolution'
43             elif 'R' in bill_id:
44                 btype = 'resolution'
45             elif 'B' in bill_id:
46                 btype = 'bill'
47 
48             title = bill_data['SHORTTITLE'] or bill_data['LONGTITLE']
49 
50             # main
51             bill = Bill(
52                 bill_id,
53                 session,
54                 title,
55                 chamber=chamber,
56                 classification=btype,
57             )
58             bill.extras = {'status': bill_data['STATUS']}
59 
60             bill.add_source(ksapi.url + 'bill_status/' + bill_id.lower())
61 
62             if (bill_data['LONGTITLE'] and
63                     bill_data['LONGTITLE'] != bill.title):
64                 bill.add_title(bill_data['LONGTITLE'])
65 
66             # An "original sponsor" is the API's expression of "primary sponsor"
67             for primary_sponsor in bill_data['ORIGINAL_SPONSOR']:
68                 bill.add_sponsorship(
69                     name=primary_sponsor,
70                     entity_type='organization' if "committee" in primary_sponsor.lower()
71                                 else 'person',
72                     primary=True,
73                     classification="original sponsor"
74                 )
75             for sponsor in bill_data['SPONSOR_NAMES']:
76                 if sponsor in bill_data['ORIGINAL_SPONSOR']:
77                     continue
78                 bill.add_sponsorship(
79                     name=sponsor,
80                     entity_type='organization' if "committee" in sponsor.lower() else 'person',
81                     primary=False,
82                     classification='cosponsor',
83                 )
84 
85             # history is backwards
86             for event in reversed(bill_data['HISTORY']):
87                 actor = ('upper' if event['chamber'] == 'Senate'
88                          else 'lower')
89 
90                 date = event['session_date']
91                 # append committee names if present
92                 if 'committee_names' in event:
93                     action = (event['status'] + ' ' +
94                               ' and '.join(event['committee_names']))
95                 else:
96                     action = event['status']
97 
98                 if event['action_code'] not in ksapi.action_codes:
99                     self.warning('unknown action code on %s: %s %s' %
100                                  (bill_id, event['action_code'],
101                                   event['status']))
102                     atype = None
103                 else:
104                     atype = ksapi.action_codes[event['action_code']]
105                 bill.add_action(
106                     action, date, chamber=actor, classification=atype)
107 
108             # Versions are exposed in `bill_data['versions'],
109             # but lack any descriptive text or identifiers;
110             # continue to scrape these from the HTML
111             yield from self.scrape_html(bill, session)
112 
113             yield bill
114 
115     def scrape_html(self, bill, session):
116         meta = next(
117             each for each in self.jurisdiction.legislative_sessions
118             if each['identifier'] == session
119         )
120         slug = meta['_scraped_name']
121         # we have to go to the HTML for the versions & votes
122         base_url = 'http://www.kslegislature.org/li/%s/measures/' % slug
123         if 'resolution' in bill.classification:
124             base_url = 'http://www.kslegislature.org/li/%s/year1/measures/' % slug
125 
126         url = base_url + bill.identifier.lower() + '/'
127         doc = lxml.html.fromstring(self.get(url).text)
128         doc.make_links_absolute(url)
129 
130         bill.add_source(url)
131 
132         # versions & notes
133         version_rows = doc.xpath('//tbody[starts-with(@id, "version-tab")]/tr')
134         for row in version_rows:
135             # version, docs, sn, fn
136             tds = row.getchildren()
137             title = _clean_spaces(tds[0].text_content().strip())
138             doc_url = get_doc_link(tds[1])
139             if doc_url:
140                 bill.add_version_link(title, doc_url, media_type='application/pdf')
141             if len(tds) > 2:
142                 sn_url = get_doc_link(tds[2])
143                 if sn_url:
144                     bill.add_document_link(
145                         title + ' - Supplementary Note', sn_url,
146                         on_duplicate='ignore'
147                     )
148             if len(tds) > 3:
149                 if sn_url:
150                     bill.add_document_link(title + ' - Fiscal Note', sn_url,
151                                            on_duplicate='ignore'
152                                            )
153 
154         all_links = doc.xpath(
155             "//table[@class='bottom']/tbody[@class='tab-content-sub']/tr/td/a/@href"
156         )
157         vote_members_urls = []
158         for i in all_links:
159             if "vote_view" in i:
160                 vote_members_urls.append(str(i))
161         if len(vote_members_urls) > 0:
162             for link in vote_members_urls:
163                 yield from self.parse_vote(bill, link)
164 
165         history_rows = doc.xpath('//tbody[starts-with(@id, "history-tab")]/tr')
166         for row in history_rows:
167             row_text = row.xpath('.//td[3]')[0].text_content()
168             # amendments & reports
169             amendment = get_doc_link(row.xpath('.//td[4]')[0])
170             if amendment:
171                 if 'Motion to Amend' in row_text:
172                     _, offered_by = row_text.split('Motion to Amend -')
173                     amendment_name = 'Amendment ' + offered_by.strip()
174                 elif 'Conference committee report now available' in row_text:
175                     amendment_name = 'Conference Committee Report'
176                 else:
177                     amendment_name = row_text.strip()
178                 bill.add_document_link(_clean_spaces(amendment_name), amendment,
179                                        on_duplicate='ignore')
180 
181     def parse_vote(self, bill, link):
182         text = self.get(link).text
183         if 'Page Not Found' in text:
184             self.warning("missing vote, skipping")
185             return
186         member_doc = lxml.html.fromstring(text)
187         motion = member_doc.xpath("//div[@id='main_content']/h4/text()")
188         chamber_date_line = ''.join(member_doc.xpath("//div[@id='main_content']/h3[1]//text()"))
189         chamber_date_line_words = chamber_date_line.split()
190         vote_chamber = chamber_date_line_words[0]
191         vote_date = datetime.datetime.strptime(chamber_date_line_words[-1], '%m/%d/%Y')
192         vote_status = " ".join(chamber_date_line_words[2:-2])
193 
194         opinions = member_doc.xpath("//div[@id='main_content']/h3[position() > 1]/text()")
195         if len(opinions) > 0:
196             vote_status = vote_status if vote_status.strip() else motion[0]
197             vote_chamber = 'upper' if vote_chamber == 'Senate' else 'lower'
198 
199             for i in opinions:
200                 try:
201                     count = int(i[i.find("(") + 1:i.find(")")])
202                 except ValueError:
203                     # This is likely not a vote-count text chunk
204                     # It's probably '`On roll call the vote was:`
205                     pass
206                 else:
207                     if "yea" in i.lower():
208                         yes_count = count
209                     elif "nay" in i.lower():
210                         no_count = count
211                     elif "present" in i.lower():
212                         p_count = count
213                     elif "absent" in i.lower():
214                         a_count = count
215 
216             vote = VoteEvent(
217                 bill=bill,
218                 start_date=vote_date.strftime('%Y-%m-%d'),
219                 chamber=vote_chamber,
220                 motion_text=vote_status,
221                 result='pass' if yes_count > no_count else 'fail',
222                 classification='passage',
223             )
224             vote.pupa_id = link
225 
226             vote.set_count('yes', yes_count)
227             vote.set_count('no', no_count)
228             vote.set_count('abstain', p_count)
229             vote.set_count('absent', a_count)
230 
231             vote.add_source(link)
232 
233             a_links = member_doc.xpath("//div[@id='main_content']/a/text()")
234             for i in range(1, len(a_links)):
235                 if i <= yes_count:
236                     vote.vote('yes', re.sub(',', '', a_links[i]).split()[0])
237                 elif no_count != 0 and i > yes_count and i <= yes_count + no_count:
238                     vote.vote('no', re.sub(',', '', a_links[i]).split()[0])
239                 else:
240                     vote.vote('other', re.sub(',', '', a_links[i]).split()[0])
241             yield vote
242         else:
243             self.warning("No Votes for: %s", link)
244 
245 
246 def get_doc_link(elem):
247     # try ODT then PDF
248     link = elem.xpath('.//a[contains(@href, ".odt")]/@href')
249     if link:
250         return link[0]
251     link = elem.xpath('.//a[contains(@href, ".pdf")]/@href')
252     if link:
253         return link[0]
254
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/openstates/ks/bills.py b/openstates/ks/bills.py
--- a/openstates/ks/bills.py
+++ b/openstates/ks/bills.py
@@ -1,6 +1,7 @@
 import re
 import json
 import datetime
+import requests
 
 import lxml.html
 from pupa.scrape import Scraper, Bill, VoteEvent
@@ -179,8 +180,16 @@
                                        on_duplicate='ignore')
 
     def parse_vote(self, bill, link):
-        text = self.get(link).text
-        if 'Page Not Found' in text:
+        # Server sometimes sends proper error headers,
+        # sometimes not
+        try:
+            self.info("Get {}".format(link))
+            text = requests.get(link).text
+        except requests.exceptions.HTTPError as err:
+            self.warning("{} fetching vote {}, skipping".format(err, link))
+            return
+
+        if 'Page Not Found' in text or 'Page Unavailable' in text:
             self.warning("missing vote, skipping")
             return
         member_doc = lxml.html.fromstring(text)

{"golden_diff": "diff --git a/openstates/ks/bills.py b/openstates/ks/bills.py\n--- a/openstates/ks/bills.py\n+++ b/openstates/ks/bills.py\n@@ -1,6 +1,7 @@\n import re\n import json\n import datetime\n+import requests\n \n import lxml.html\n from pupa.scrape import Scraper, Bill, VoteEvent\n@@ -179,8 +180,16 @@\n                                        on_duplicate='ignore')\n \n     def parse_vote(self, bill, link):\n-        text = self.get(link).text\n-        if 'Page Not Found' in text:\n+        # Server sometimes sends proper error headers,\n+        # sometimes not\n+        try:\n+            self.info(\"Get {}\".format(link))\n+            text = requests.get(link).text\n+        except requests.exceptions.HTTPError as err:\n+            self.warning(\"{} fetching vote {}, skipping\".format(err, link))\n+            return\n+\n+        if 'Page Not Found' in text or 'Page Unavailable' in text:\n             self.warning(\"missing vote, skipping\")\n             return\n         member_doc = lxml.html.fromstring(text)\n", "issue": "KS failing since at least 2019-02-24\nKS has been failing since 2019-02-24\n\nBased on automated runs it appears that KS has not run successfully in 2 days (2019-02-24).\n\n\n```\n  06:02:22 INFO pupa: save jurisdiction Kansas as jurisdiction_ocd-jurisdiction-country:us-state:ks-government.json\n06:02:22 INFO pupa: save organization Kansas State Legislature as organization_ea95774a-38c2-11e9-b4ad-021e9c8c6450.json\n06:02:22 INFO pupa: save organization Senate as organization_ea957a1a-38c2-11e9-b4ad-021e9c8c6450.json\n06:02:22 INFO pupa: save organization House as organization_ea957bbe-38c2-11e9-b4ad-021e9c8c6450.json\n06:02:22 INFO pupa: no session specified, using 2019-2020\n06:02:22 INFO scrapelib: GET - http://www.kslegislature.org/li/api/v11/rev-1/bill_status/\n06:02:23 WARNING pupa: unknown action code on SB9: misc_he_200 Hearing: Tuesday, January 22, 2019, 10:30 AM Room 548-S\n06:02:23 WARNING pupa: unknown action code on SB9: misc_he_200 Hearing: Tuesday, February 12, 2019, 9:00 AM Room 112-N\n06:02:23 INFO scrapelib: GET - http://www.kslegislature.org/li/b2019_20/measures/sb9/\n06:02:27 INFO scrapelib: GET - http://www.kslegislature.org/li/b2019_20/measures/vote_view/je_20190222111337_914158/\n06:02:32 WARNING scrapelib: sleeping for 10 seconds before retry\n06:02:42 WARNING scrapelib: sleeping for 20 seconds before retry\n06:03:03 WARNING scrapelib: sleeping for 40 seconds before retry\n```\n\nVisit http://bobsled.openstates.org for more info.\n\n", "before_files": [{"content": "import re\nimport json\nimport datetime\n\nimport lxml.html\nfrom pupa.scrape import Scraper, Bill, VoteEvent\n\nfrom . import ksapi\n\n\ndef _clean_spaces(title):\n    return re.sub(r'\\s+', ' ', title)\n\n\nclass KSBillScraper(Scraper):\n    def scrape(self, chamber=None, session=None):\n        if session is None:\n            session = self.latest_session()\n            self.info('no session specified, using %s', session)\n\n        chambers = [chamber] if chamber is not None else ['upper', 'lower']\n\n        for chamber in chambers:\n            yield from self.scrape_chamber(chamber, session)\n\n    def scrape_chamber(self, chamber, session):\n        chamber_name = 'Senate' if chamber == 'upper' else 'House'\n        chamber_letter = chamber_name[0]\n        # perhaps we should save this data so we can make one request for both?\n        bill_request = self.get(ksapi.url + 'bill_status/').text\n        bill_request_json = json.loads(bill_request)\n        bills = bill_request_json['content']\n        for bill_data in bills:\n\n            bill_id = bill_data['BILLNO']\n\n            # filter other chambers\n            if not bill_id.startswith(chamber_letter):\n                continue\n\n            if 'CR' in bill_id:\n                btype = 'concurrent resolution'\n            elif 'R' in bill_id:\n                btype = 'resolution'\n            elif 'B' in bill_id:\n                btype = 'bill'\n\n            title = bill_data['SHORTTITLE'] or bill_data['LONGTITLE']\n\n            # main\n            bill = Bill(\n                bill_id,\n                session,\n                title,\n                chamber=chamber,\n                classification=btype,\n            )\n            bill.extras = {'status': bill_data['STATUS']}\n\n            bill.add_source(ksapi.url + 'bill_status/' + bill_id.lower())\n\n            if (bill_data['LONGTITLE'] and\n                    bill_data['LONGTITLE'] != bill.title):\n                bill.add_title(bill_data['LONGTITLE'])\n\n            # An \"original sponsor\" is the API's expression of \"primary sponsor\"\n            for primary_sponsor in bill_data['ORIGINAL_SPONSOR']:\n                bill.add_sponsorship(\n                    name=primary_sponsor,\n                    entity_type='organization' if \"committee\" in primary_sponsor.lower()\n                                else 'person',\n                    primary=True,\n                    classification=\"original sponsor\"\n                )\n            for sponsor in bill_data['SPONSOR_NAMES']:\n                if sponsor in bill_data['ORIGINAL_SPONSOR']:\n                    continue\n                bill.add_sponsorship(\n                    name=sponsor,\n                    entity_type='organization' if \"committee\" in sponsor.lower() else 'person',\n                    primary=False,\n                    classification='cosponsor',\n                )\n\n            # history is backwards\n            for event in reversed(bill_data['HISTORY']):\n                actor = ('upper' if event['chamber'] == 'Senate'\n                         else 'lower')\n\n                date = event['session_date']\n                # append committee names if present\n                if 'committee_names' in event:\n                    action = (event['status'] + ' ' +\n                              ' and '.join(event['committee_names']))\n                else:\n                    action = event['status']\n\n                if event['action_code'] not in ksapi.action_codes:\n                    self.warning('unknown action code on %s: %s %s' %\n                                 (bill_id, event['action_code'],\n                                  event['status']))\n                    atype = None\n                else:\n                    atype = ksapi.action_codes[event['action_code']]\n                bill.add_action(\n                    action, date, chamber=actor, classification=atype)\n\n            # Versions are exposed in `bill_data['versions'],\n            # but lack any descriptive text or identifiers;\n            # continue to scrape these from the HTML\n            yield from self.scrape_html(bill, session)\n\n            yield bill\n\n    def scrape_html(self, bill, session):\n        meta = next(\n            each for each in self.jurisdiction.legislative_sessions\n            if each['identifier'] == session\n        )\n        slug = meta['_scraped_name']\n        # we have to go to the HTML for the versions & votes\n        base_url = 'http://www.kslegislature.org/li/%s/measures/' % slug\n        if 'resolution' in bill.classification:\n            base_url = 'http://www.kslegislature.org/li/%s/year1/measures/' % slug\n\n        url = base_url + bill.identifier.lower() + '/'\n        doc = lxml.html.fromstring(self.get(url).text)\n        doc.make_links_absolute(url)\n\n        bill.add_source(url)\n\n        # versions & notes\n        version_rows = doc.xpath('//tbody[starts-with(@id, \"version-tab\")]/tr')\n        for row in version_rows:\n            # version, docs, sn, fn\n            tds = row.getchildren()\n            title = _clean_spaces(tds[0].text_content().strip())\n            doc_url = get_doc_link(tds[1])\n            if doc_url:\n                bill.add_version_link(title, doc_url, media_type='application/pdf')\n            if len(tds) > 2:\n                sn_url = get_doc_link(tds[2])\n                if sn_url:\n                    bill.add_document_link(\n                        title + ' - Supplementary Note', sn_url,\n                        on_duplicate='ignore'\n                    )\n            if len(tds) > 3:\n                if sn_url:\n                    bill.add_document_link(title + ' - Fiscal Note', sn_url,\n                                           on_duplicate='ignore'\n                                           )\n\n        all_links = doc.xpath(\n            \"//table[@class='bottom']/tbody[@class='tab-content-sub']/tr/td/a/@href\"\n        )\n        vote_members_urls = []\n        for i in all_links:\n            if \"vote_view\" in i:\n                vote_members_urls.append(str(i))\n        if len(vote_members_urls) > 0:\n            for link in vote_members_urls:\n                yield from self.parse_vote(bill, link)\n\n        history_rows = doc.xpath('//tbody[starts-with(@id, \"history-tab\")]/tr')\n        for row in history_rows:\n            row_text = row.xpath('.//td[3]')[0].text_content()\n            # amendments & reports\n            amendment = get_doc_link(row.xpath('.//td[4]')[0])\n            if amendment:\n                if 'Motion to Amend' in row_text:\n                    _, offered_by = row_text.split('Motion to Amend -')\n                    amendment_name = 'Amendment ' + offered_by.strip()\n                elif 'Conference committee report now available' in row_text:\n                    amendment_name = 'Conference Committee Report'\n                else:\n                    amendment_name = row_text.strip()\n                bill.add_document_link(_clean_spaces(amendment_name), amendment,\n                                       on_duplicate='ignore')\n\n    def parse_vote(self, bill, link):\n        text = self.get(link).text\n        if 'Page Not Found' in text:\n            self.warning(\"missing vote, skipping\")\n            return\n        member_doc = lxml.html.fromstring(text)\n        motion = member_doc.xpath(\"//div[@id='main_content']/h4/text()\")\n        chamber_date_line = ''.join(member_doc.xpath(\"//div[@id='main_content']/h3[1]//text()\"))\n        chamber_date_line_words = chamber_date_line.split()\n        vote_chamber = chamber_date_line_words[0]\n        vote_date = datetime.datetime.strptime(chamber_date_line_words[-1], '%m/%d/%Y')\n        vote_status = \" \".join(chamber_date_line_words[2:-2])\n\n        opinions = member_doc.xpath(\"//div[@id='main_content']/h3[position() > 1]/text()\")\n        if len(opinions) > 0:\n            vote_status = vote_status if vote_status.strip() else motion[0]\n            vote_chamber = 'upper' if vote_chamber == 'Senate' else 'lower'\n\n            for i in opinions:\n                try:\n                    count = int(i[i.find(\"(\") + 1:i.find(\")\")])\n                except ValueError:\n                    # This is likely not a vote-count text chunk\n                    # It's probably '`On roll call the vote was:`\n                    pass\n                else:\n                    if \"yea\" in i.lower():\n                        yes_count = count\n                    elif \"nay\" in i.lower():\n                        no_count = count\n                    elif \"present\" in i.lower():\n                        p_count = count\n                    elif \"absent\" in i.lower():\n                        a_count = count\n\n            vote = VoteEvent(\n                bill=bill,\n                start_date=vote_date.strftime('%Y-%m-%d'),\n                chamber=vote_chamber,\n                motion_text=vote_status,\n                result='pass' if yes_count > no_count else 'fail',\n                classification='passage',\n            )\n            vote.pupa_id = link\n\n            vote.set_count('yes', yes_count)\n            vote.set_count('no', no_count)\n            vote.set_count('abstain', p_count)\n            vote.set_count('absent', a_count)\n\n            vote.add_source(link)\n\n            a_links = member_doc.xpath(\"//div[@id='main_content']/a/text()\")\n            for i in range(1, len(a_links)):\n                if i <= yes_count:\n                    vote.vote('yes', re.sub(',', '', a_links[i]).split()[0])\n                elif no_count != 0 and i > yes_count and i <= yes_count + no_count:\n                    vote.vote('no', re.sub(',', '', a_links[i]).split()[0])\n                else:\n                    vote.vote('other', re.sub(',', '', a_links[i]).split()[0])\n            yield vote\n        else:\n            self.warning(\"No Votes for: %s\", link)\n\n\ndef get_doc_link(elem):\n    # try ODT then PDF\n    link = elem.xpath('.//a[contains(@href, \".odt\")]/@href')\n    if link:\n        return link[0]\n    link = elem.xpath('.//a[contains(@href, \".pdf\")]/@href')\n    if link:\n        return link[0]\n", "path": "openstates/ks/bills.py"}], "after_files": [{"content": "import re\nimport json\nimport datetime\nimport requests\n\nimport lxml.html\nfrom pupa.scrape import Scraper, Bill, VoteEvent\n\nfrom . import ksapi\n\n\ndef _clean_spaces(title):\n    return re.sub(r'\\s+', ' ', title)\n\n\nclass KSBillScraper(Scraper):\n    def scrape(self, chamber=None, session=None):\n        if session is None:\n            session = self.latest_session()\n            self.info('no session specified, using %s', session)\n\n        chambers = [chamber] if chamber is not None else ['upper', 'lower']\n\n        for chamber in chambers:\n            yield from self.scrape_chamber(chamber, session)\n\n    def scrape_chamber(self, chamber, session):\n        chamber_name = 'Senate' if chamber == 'upper' else 'House'\n        chamber_letter = chamber_name[0]\n        # perhaps we should save this data so we can make one request for both?\n        bill_request = self.get(ksapi.url + 'bill_status/').text\n        bill_request_json = json.loads(bill_request)\n        bills = bill_request_json['content']\n        for bill_data in bills:\n\n            bill_id = bill_data['BILLNO']\n\n            # filter other chambers\n            if not bill_id.startswith(chamber_letter):\n                continue\n\n            if 'CR' in bill_id:\n                btype = 'concurrent resolution'\n            elif 'R' in bill_id:\n                btype = 'resolution'\n            elif 'B' in bill_id:\n                btype = 'bill'\n\n            title = bill_data['SHORTTITLE'] or bill_data['LONGTITLE']\n\n            # main\n            bill = Bill(\n                bill_id,\n                session,\n                title,\n                chamber=chamber,\n                classification=btype,\n            )\n            bill.extras = {'status': bill_data['STATUS']}\n\n            bill.add_source(ksapi.url + 'bill_status/' + bill_id.lower())\n\n            if (bill_data['LONGTITLE'] and\n                    bill_data['LONGTITLE'] != bill.title):\n                bill.add_title(bill_data['LONGTITLE'])\n\n            # An \"original sponsor\" is the API's expression of \"primary sponsor\"\n            for primary_sponsor in bill_data['ORIGINAL_SPONSOR']:\n                bill.add_sponsorship(\n                    name=primary_sponsor,\n                    entity_type='organization' if \"committee\" in primary_sponsor.lower()\n                                else 'person',\n                    primary=True,\n                    classification=\"original sponsor\"\n                )\n            for sponsor in bill_data['SPONSOR_NAMES']:\n                if sponsor in bill_data['ORIGINAL_SPONSOR']:\n                    continue\n                bill.add_sponsorship(\n                    name=sponsor,\n                    entity_type='organization' if \"committee\" in sponsor.lower() else 'person',\n                    primary=False,\n                    classification='cosponsor',\n                )\n\n            # history is backwards\n            for event in reversed(bill_data['HISTORY']):\n                actor = ('upper' if event['chamber'] == 'Senate'\n                         else 'lower')\n\n                date = event['session_date']\n                # append committee names if present\n                if 'committee_names' in event:\n                    action = (event['status'] + ' ' +\n                              ' and '.join(event['committee_names']))\n                else:\n                    action = event['status']\n\n                if event['action_code'] not in ksapi.action_codes:\n                    self.warning('unknown action code on %s: %s %s' %\n                                 (bill_id, event['action_code'],\n                                  event['status']))\n                    atype = None\n                else:\n                    atype = ksapi.action_codes[event['action_code']]\n                bill.add_action(\n                    action, date, chamber=actor, classification=atype)\n\n            # Versions are exposed in `bill_data['versions'],\n            # but lack any descriptive text or identifiers;\n            # continue to scrape these from the HTML\n            yield from self.scrape_html(bill, session)\n\n            yield bill\n\n    def scrape_html(self, bill, session):\n        meta = next(\n            each for each in self.jurisdiction.legislative_sessions\n            if each['identifier'] == session\n        )\n        slug = meta['_scraped_name']\n        # we have to go to the HTML for the versions & votes\n        base_url = 'http://www.kslegislature.org/li/%s/measures/' % slug\n        if 'resolution' in bill.classification:\n            base_url = 'http://www.kslegislature.org/li/%s/year1/measures/' % slug\n\n        url = base_url + bill.identifier.lower() + '/'\n        doc = lxml.html.fromstring(self.get(url).text)\n        doc.make_links_absolute(url)\n\n        bill.add_source(url)\n\n        # versions & notes\n        version_rows = doc.xpath('//tbody[starts-with(@id, \"version-tab\")]/tr')\n        for row in version_rows:\n            # version, docs, sn, fn\n            tds = row.getchildren()\n            title = _clean_spaces(tds[0].text_content().strip())\n            doc_url = get_doc_link(tds[1])\n            if doc_url:\n                bill.add_version_link(title, doc_url, media_type='application/pdf')\n            if len(tds) > 2:\n                sn_url = get_doc_link(tds[2])\n                if sn_url:\n                    bill.add_document_link(\n                        title + ' - Supplementary Note', sn_url,\n                        on_duplicate='ignore'\n                    )\n            if len(tds) > 3:\n                if sn_url:\n                    bill.add_document_link(title + ' - Fiscal Note', sn_url,\n                                           on_duplicate='ignore'\n                                           )\n\n        all_links = doc.xpath(\n            \"//table[@class='bottom']/tbody[@class='tab-content-sub']/tr/td/a/@href\"\n        )\n        vote_members_urls = []\n        for i in all_links:\n            if \"vote_view\" in i:\n                vote_members_urls.append(str(i))\n        if len(vote_members_urls) > 0:\n            for link in vote_members_urls:\n                yield from self.parse_vote(bill, link)\n\n        history_rows = doc.xpath('//tbody[starts-with(@id, \"history-tab\")]/tr')\n        for row in history_rows:\n            row_text = row.xpath('.//td[3]')[0].text_content()\n            # amendments & reports\n            amendment = get_doc_link(row.xpath('.//td[4]')[0])\n            if amendment:\n                if 'Motion to Amend' in row_text:\n                    _, offered_by = row_text.split('Motion to Amend -')\n                    amendment_name = 'Amendment ' + offered_by.strip()\n                elif 'Conference committee report now available' in row_text:\n                    amendment_name = 'Conference Committee Report'\n                else:\n                    amendment_name = row_text.strip()\n                bill.add_document_link(_clean_spaces(amendment_name), amendment,\n                                       on_duplicate='ignore')\n\n    def parse_vote(self, bill, link):\n        # Server sometimes sends proper error headers,\n        # sometimes not\n        try:\n            self.info(\"Get {}\".format(link))\n            text = requests.get(link).text\n        except requests.exceptions.HTTPError as err:\n            self.warning(\"{} fetching vote {}, skipping\".format(err, link))\n            return\n\n        if 'Page Not Found' in text or 'Page Unavailable' in text:\n            self.warning(\"missing vote, skipping\")\n            return\n        member_doc = lxml.html.fromstring(text)\n        motion = member_doc.xpath(\"//div[@id='main_content']/h4/text()\")\n        chamber_date_line = ''.join(member_doc.xpath(\"//div[@id='main_content']/h3[1]//text()\"))\n        chamber_date_line_words = chamber_date_line.split()\n        vote_chamber = chamber_date_line_words[0]\n        vote_date = datetime.datetime.strptime(chamber_date_line_words[-1], '%m/%d/%Y')\n        vote_status = \" \".join(chamber_date_line_words[2:-2])\n\n        opinions = member_doc.xpath(\"//div[@id='main_content']/h3[position() > 1]/text()\")\n        if len(opinions) > 0:\n            vote_status = vote_status if vote_status.strip() else motion[0]\n            vote_chamber = 'upper' if vote_chamber == 'Senate' else 'lower'\n\n            for i in opinions:\n                try:\n                    count = int(i[i.find(\"(\") + 1:i.find(\")\")])\n                except ValueError:\n                    # This is likely not a vote-count text chunk\n                    # It's probably '`On roll call the vote was:`\n                    pass\n                else:\n                    if \"yea\" in i.lower():\n                        yes_count = count\n                    elif \"nay\" in i.lower():\n                        no_count = count\n                    elif \"present\" in i.lower():\n                        p_count = count\n                    elif \"absent\" in i.lower():\n                        a_count = count\n\n            vote = VoteEvent(\n                bill=bill,\n                start_date=vote_date.strftime('%Y-%m-%d'),\n                chamber=vote_chamber,\n                motion_text=vote_status,\n                result='pass' if yes_count > no_count else 'fail',\n                classification='passage',\n            )\n            vote.pupa_id = link\n\n            vote.set_count('yes', yes_count)\n            vote.set_count('no', no_count)\n            vote.set_count('abstain', p_count)\n            vote.set_count('absent', a_count)\n\n            vote.add_source(link)\n\n            a_links = member_doc.xpath(\"//div[@id='main_content']/a/text()\")\n            for i in range(1, len(a_links)):\n                if i <= yes_count:\n                    vote.vote('yes', re.sub(',', '', a_links[i]).split()[0])\n                elif no_count != 0 and i > yes_count and i <= yes_count + no_count:\n                    vote.vote('no', re.sub(',', '', a_links[i]).split()[0])\n                else:\n                    vote.vote('other', re.sub(',', '', a_links[i]).split()[0])\n            yield vote\n        else:\n            self.warning(\"No Votes for: %s\", link)\n\n\ndef get_doc_link(elem):\n    # try ODT then PDF\n    link = elem.xpath('.//a[contains(@href, \".odt\")]/@href')\n    if link:\n        return link[0]\n    link = elem.xpath('.//a[contains(@href, \".pdf\")]/@href')\n    if link:\n        return link[0]\n", "path": "openstates/ks/bills.py"}]}