We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
AR scraper failing since at least 2017-03-11
State: AR - scraper has been failing since 2017-03-11

Based on automated runs it appears that AR has not run successfully in 2 days (2017-03-11).

```/usr/local/bin/billy-update ar``` | **failed during bills**

```
  Traceback (most recent call last):
  File "/opt/openstates/billy/billy/bin/update.py", line 368, in main
    run_record += _run_scraper(stype, args, metadata)
  File "/opt/openstates/billy/billy/bin/update.py", line 102, in _run_scraper
    scraper.scrape(chamber, time)
  File "/srv/openstates-web/openstates/ar/bills.py", line 40, in scrape
    self.save_bill(bill)
  File "/opt/openstates/billy/billy/scrape/__init__.py", line 199, in save_object
    self.validate_json(obj)
  File "/opt/openstates/billy/billy/scrape/__init__.py", line 130, in validate_json
    raise ve
FieldValidationError: Value u'' for field '<obj>.sponsors[0].name' cannot be blank'

```

Visit http://bobsled.openstates.org/ for more info.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `openstates/ar/bills.py`
Content:
```
1 import re
2 import csv
3 import StringIO
4 import datetime
5 
6 from billy.scrape.bills import BillScraper, Bill
7 from billy.scrape.votes import Vote
8 
9 import lxml.html
10 
11 import scrapelib
12 
13 
14 def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
15     # csv.py doesn't do Unicode; encode temporarily as UTF-8:
16     csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
17                             dialect=dialect, **kwargs)
18     for row in csv_reader:
19         # decode UTF-8 back to Unicode, cell by cell:
20         yield [unicode(cell, 'utf-8') for cell in row]
21 
22 
23 def utf_8_encoder(unicode_csv_data):
24     for line in unicode_csv_data:
25         yield line.encode('utf-8')
26 
27 
28 class ARBillScraper(BillScraper):
29     jurisdiction = 'ar'
30 
31     def scrape(self, chamber, session):
32         self.bills = {}
33 
34         self.slug = self.metadata['session_details'][session]['slug']
35 
36         self.scrape_bill(chamber, session)
37         self.scrape_actions()
38 
39         for bill in self.bills.itervalues():
40             self.save_bill(bill)
41 
42     def scrape_bill(self, chamber, session):
43         url = "ftp://www.arkleg.state.ar.us/dfadooas/LegislativeMeasures.txt"
44         page = self.get(url).text
45         page = unicode_csv_reader(StringIO.StringIO(page), delimiter='|')
46 
47         for row in page:
48             bill_chamber = {'H': 'lower', 'S': 'upper'}[row[0]]
49             if bill_chamber != chamber:
50                 continue
51 
52             bill_id = "%s%s %s" % (row[0], row[1], row[2])
53 
54             type_spec = re.match(r'(H|S)([A-Z]+)\s', bill_id).group(2)
55             bill_type = {
56                 'B': 'bill',
57                 'R': 'resolution',
58                 'JR': 'joint resolution',
59                 'CR': 'concurrent resolution',
60                 'MR': 'memorial resolution',
61                 'CMR': 'concurrent memorial resolution'}[type_spec]
62 
63             if row[-1] != self.slug:
64                 continue
65 
66             bill = Bill(session, chamber, bill_id, row[3], type=bill_type)
67             bill.add_source(url)
68 
69             primary = row[11]
70             if not primary:
71                 primary = row[12]
72             bill.add_sponsor('primary', primary)
73 
74             # ftp://www.arkleg.state.ar.us/Bills/
75             # TODO: Keep on eye on this post 2017 to see if they apply R going forward.
76             session_code = '2017R' if session == '2017' else session
77 
78             version_url = ("ftp://www.arkleg.state.ar.us/Bills/"
79                            "%s/Public/%s.pdf" % (
80                                session_code, bill_id.replace(' ', '')))
81             bill.add_version(bill_id, version_url, mimetype='application/pdf')
82 
83             self.scrape_bill_page(bill)
84 
85             self.bills[bill_id] = bill
86 
87     def scrape_actions(self):
88         url = "ftp://www.arkleg.state.ar.us/dfadooas/ChamberActions.txt"
89         page = self.get(url).text
90         page = csv.reader(StringIO.StringIO(page))
91 
92         for row in page:
93             bill_id = "%s%s %s" % (row[1], row[2], row[3])
94 
95             if bill_id not in self.bills:
96                 continue
97             # different term
98             if row[-2] != self.slug:
99                 continue
100 
101             # Commas aren't escaped, but only one field (the action) can
102             # contain them so we can work around it by using both positive
103             # and negative offsets
104             bill_id = "%s%s %s" % (row[1], row[2], row[3])
105             actor = {'HU': 'lower', 'SU': 'upper'}[row[-5].upper()]
106             # manual fix for crazy time value
107             row[6] = row[6].replace('.520000000', '')
108             date = datetime.datetime.strptime(row[6], "%Y-%m-%d %H:%M:%S")
109             action = ','.join(row[7:-5])
110 
111             action_type = []
112             if action.startswith('Filed'):
113                 action_type.append('bill:introduced')
114             elif (action.startswith('Read first time') or
115                   action.startswith('Read the first time')):
116                 action_type.append('bill:reading:1')
117             if re.match('Read the first time, .*, read the second time', action):
118                 action_type.append('bill:reading:2')
119             elif action.startswith('Read the third time and passed'):
120                 action_type.append('bill:passed')
121                 action_type.append('bill:reading:3')
122             elif action.startswith('Read the third time'):
123                 action_type.append('bill:reading:3')
124             elif action.startswith('DELIVERED TO GOVERNOR'):
125                 action_type.append('governor:received')
126             elif action.startswith('Notification'):
127                 action_type.append('governor:signed')
128 
129             if 'referred to' in action:
130                 action_type.append('committee:referred')
131 
132             if 'Returned by the Committee' in action:
133                 if 'recommendation that it Do Pass' in action:
134                     action_type.append('committee:passed:favorable')
135                 else:
136                     action_type.append('committee:passed')
137 
138             if re.match(r'Amendment No\. \d+ read and adopted', action):
139                 action_type.append('amendment:introduced')
140                 action_type.append('amendment:passed')
141 
142             if not action:
143                 action = '[No text provided]'
144             self.bills[bill_id].add_action(actor, action, date,
145                                            type=action_type or ['other'])
146 
147     def scrape_bill_page(self, bill):
148         # We need to scrape each bill page in order to grab associated votes.
149         # It's still more efficient to get the rest of the data we're
150         # interested in from the CSVs, though, because their site splits
151         # other info (e.g. actions) across many pages
152         for t in self.metadata['terms']:
153             if bill['session'] in t['sessions']:
154                 term_year = t['start_year']
155                 break
156         measureno = bill['bill_id'].replace(' ', '')
157         url = ("http://www.arkleg.state.ar.us/assembly/%s/%s/"
158                "Pages/BillInformation.aspx?measureno=%s" % (
159                    term_year, self.slug, measureno))
160         bill.add_source(url)
161 
162         page = lxml.html.fromstring(self.get(url).text)
163         page.make_links_absolute(url)
164 
165         for link in page.xpath("//a[contains(@href, 'Amendments')]"):
166             num = link.xpath("string(../../td[2])")
167             name = "Amendment %s" % num
168             bill.add_document(name, link.attrib['href'])
169 
170         try:
171             cosponsor_link = page.xpath(
172                 "//a[contains(@href, 'CoSponsors')]")[0]
173             self.scrape_cosponsors(bill, cosponsor_link.attrib['href'])
174         except IndexError:
175             # No cosponsor link is OK
176             pass
177 
178     #     hist_link = page.xpath("//a[contains(@href, 'BillStatusHistory')]")[0]
179     #     self.scrape_votes(bill, hist_link.attrib['href'])
180 
181     # def scrape_votes(self, bill, url):
182     #     page = lxml.html.fromstring(self.get(url).text)
183     #     page.make_links_absolute(url)
184 
185         for link in page.xpath("//a[contains(@href, 'votes.aspx')]"):
186             date = link.xpath("string(../../td[2])")
187             date = datetime.datetime.strptime(date, "%m/%d/%Y %I:%M:%S %p")
188 
189             motion = link.xpath("string(../../td[3])")
190 
191             self.scrape_vote(bill, date, motion, link.attrib['href'])
192 
193     def scrape_vote(self, bill, date, motion, url):
194         try:
195             page = self.get(url).text
196         except scrapelib.HTTPError:
197             #sometiems the link is there but is dead
198             return
199 
200         if 'not yet official' in page:
201             # Sometimes they link to vote pages before they go live
202             return
203 
204         page = lxml.html.fromstring(page)
205 
206         if url.endswith('Senate'):
207             actor = 'upper'
208         else:
209             actor = 'lower'
210 
211         count_path = "string(//td[@align = 'center' and contains(., '%s: ')])"
212         yes_count = int(page.xpath(count_path % "Yeas").split()[-1])
213         no_count = int(page.xpath(count_path % "Nays").split()[-1])
214         other_count = int(page.xpath(count_path % "Non Voting").split()[-1])
215         other_count += int(page.xpath(count_path % "Present").split()[-1])
216 
217         passed = yes_count > no_count + other_count
218         vote = Vote(actor, date, motion, passed, yes_count,
219                     no_count, other_count)
220         vote.add_source(url)
221 
222         xpath = (
223             '//*[contains(@class, "ms-standardheader")]/'
224             'following-sibling::table')
225         divs = page.xpath(xpath)
226         votevals = 'yes no other other'.split()
227         for (voteval, div) in zip(votevals, divs):
228             for a in div.xpath('.//a'):
229                 name = a.text_content().strip()
230                 if not name:
231                     continue
232                 getattr(vote, voteval)(name)
233         bill.add_vote(vote)
234 
235     def scrape_cosponsors(self, bill, url):
236         page = self.get(url).text
237         page = lxml.html.fromstring(page)
238
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/openstates/ar/bills.py b/openstates/ar/bills.py
--- a/openstates/ar/bills.py
+++ b/openstates/ar/bills.py
@@ -69,7 +69,8 @@
             primary = row[11]
             if not primary:
                 primary = row[12]
-            bill.add_sponsor('primary', primary)
+            if primary:
+                bill.add_sponsor('primary', primary)
 
             # ftp://www.arkleg.state.ar.us/Bills/
             # TODO: Keep on eye on this post 2017 to see if they apply R going forward.

{"golden_diff": "diff --git a/openstates/ar/bills.py b/openstates/ar/bills.py\n--- a/openstates/ar/bills.py\n+++ b/openstates/ar/bills.py\n@@ -69,7 +69,8 @@\n             primary = row[11]\n             if not primary:\n                 primary = row[12]\n-            bill.add_sponsor('primary', primary)\n+            if primary:\n+                bill.add_sponsor('primary', primary)\n \n             # ftp://www.arkleg.state.ar.us/Bills/\n             # TODO: Keep on eye on this post 2017 to see if they apply R going forward.\n", "issue": "AR scraper failing since at least 2017-03-11\nState: AR - scraper has been failing since 2017-03-11\n\nBased on automated runs it appears that AR has not run successfully in 2 days (2017-03-11).\n\n```/usr/local/bin/billy-update ar``` | **failed during bills**\n\n```\n  Traceback (most recent call last):\n  File \"/opt/openstates/billy/billy/bin/update.py\", line 368, in main\n    run_record += _run_scraper(stype, args, metadata)\n  File \"/opt/openstates/billy/billy/bin/update.py\", line 102, in _run_scraper\n    scraper.scrape(chamber, time)\n  File \"/srv/openstates-web/openstates/ar/bills.py\", line 40, in scrape\n    self.save_bill(bill)\n  File \"/opt/openstates/billy/billy/scrape/__init__.py\", line 199, in save_object\n    self.validate_json(obj)\n  File \"/opt/openstates/billy/billy/scrape/__init__.py\", line 130, in validate_json\n    raise ve\nFieldValidationError: Value u'' for field '<obj>.sponsors[0].name' cannot be blank'\n\n```\n\nVisit http://bobsled.openstates.org/ for more info.\n\n", "before_files": [{"content": "import re\nimport csv\nimport StringIO\nimport datetime\n\nfrom billy.scrape.bills import BillScraper, Bill\nfrom billy.scrape.votes import Vote\n\nimport lxml.html\n\nimport scrapelib\n\n\ndef unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):\n    # csv.py doesn't do Unicode; encode temporarily as UTF-8:\n    csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),\n                            dialect=dialect, **kwargs)\n    for row in csv_reader:\n        # decode UTF-8 back to Unicode, cell by cell:\n        yield [unicode(cell, 'utf-8') for cell in row]\n\n\ndef utf_8_encoder(unicode_csv_data):\n    for line in unicode_csv_data:\n        yield line.encode('utf-8')\n\n\nclass ARBillScraper(BillScraper):\n    jurisdiction = 'ar'\n\n    def scrape(self, chamber, session):\n        self.bills = {}\n\n        self.slug = self.metadata['session_details'][session]['slug']\n\n        self.scrape_bill(chamber, session)\n        self.scrape_actions()\n\n        for bill in self.bills.itervalues():\n            self.save_bill(bill)\n\n    def scrape_bill(self, chamber, session):\n        url = \"ftp://www.arkleg.state.ar.us/dfadooas/LegislativeMeasures.txt\"\n        page = self.get(url).text\n        page = unicode_csv_reader(StringIO.StringIO(page), delimiter='|')\n\n        for row in page:\n            bill_chamber = {'H': 'lower', 'S': 'upper'}[row[0]]\n            if bill_chamber != chamber:\n                continue\n\n            bill_id = \"%s%s %s\" % (row[0], row[1], row[2])\n\n            type_spec = re.match(r'(H|S)([A-Z]+)\\s', bill_id).group(2)\n            bill_type = {\n                'B': 'bill',\n                'R': 'resolution',\n                'JR': 'joint resolution',\n                'CR': 'concurrent resolution',\n                'MR': 'memorial resolution',\n                'CMR': 'concurrent memorial resolution'}[type_spec]\n\n            if row[-1] != self.slug:\n                continue\n\n            bill = Bill(session, chamber, bill_id, row[3], type=bill_type)\n            bill.add_source(url)\n\n            primary = row[11]\n            if not primary:\n                primary = row[12]\n            bill.add_sponsor('primary', primary)\n\n            # ftp://www.arkleg.state.ar.us/Bills/\n            # TODO: Keep on eye on this post 2017 to see if they apply R going forward.\n            session_code = '2017R' if session == '2017' else session\n\n            version_url = (\"ftp://www.arkleg.state.ar.us/Bills/\"\n                           \"%s/Public/%s.pdf\" % (\n                               session_code, bill_id.replace(' ', '')))\n            bill.add_version(bill_id, version_url, mimetype='application/pdf')\n\n            self.scrape_bill_page(bill)\n\n            self.bills[bill_id] = bill\n\n    def scrape_actions(self):\n        url = \"ftp://www.arkleg.state.ar.us/dfadooas/ChamberActions.txt\"\n        page = self.get(url).text\n        page = csv.reader(StringIO.StringIO(page))\n\n        for row in page:\n            bill_id = \"%s%s %s\" % (row[1], row[2], row[3])\n\n            if bill_id not in self.bills:\n                continue\n            # different term\n            if row[-2] != self.slug:\n                continue\n\n            # Commas aren't escaped, but only one field (the action) can\n            # contain them so we can work around it by using both positive\n            # and negative offsets\n            bill_id = \"%s%s %s\" % (row[1], row[2], row[3])\n            actor = {'HU': 'lower', 'SU': 'upper'}[row[-5].upper()]\n            # manual fix for crazy time value\n            row[6] = row[6].replace('.520000000', '')\n            date = datetime.datetime.strptime(row[6], \"%Y-%m-%d %H:%M:%S\")\n            action = ','.join(row[7:-5])\n\n            action_type = []\n            if action.startswith('Filed'):\n                action_type.append('bill:introduced')\n            elif (action.startswith('Read first time') or\n                  action.startswith('Read the first time')):\n                action_type.append('bill:reading:1')\n            if re.match('Read the first time, .*, read the second time', action):\n                action_type.append('bill:reading:2')\n            elif action.startswith('Read the third time and passed'):\n                action_type.append('bill:passed')\n                action_type.append('bill:reading:3')\n            elif action.startswith('Read the third time'):\n                action_type.append('bill:reading:3')\n            elif action.startswith('DELIVERED TO GOVERNOR'):\n                action_type.append('governor:received')\n            elif action.startswith('Notification'):\n                action_type.append('governor:signed')\n\n            if 'referred to' in action:\n                action_type.append('committee:referred')\n\n            if 'Returned by the Committee' in action:\n                if 'recommendation that it Do Pass' in action:\n                    action_type.append('committee:passed:favorable')\n                else:\n                    action_type.append('committee:passed')\n\n            if re.match(r'Amendment No\\. \\d+ read and adopted', action):\n                action_type.append('amendment:introduced')\n                action_type.append('amendment:passed')\n\n            if not action:\n                action = '[No text provided]'\n            self.bills[bill_id].add_action(actor, action, date,\n                                           type=action_type or ['other'])\n\n    def scrape_bill_page(self, bill):\n        # We need to scrape each bill page in order to grab associated votes.\n        # It's still more efficient to get the rest of the data we're\n        # interested in from the CSVs, though, because their site splits\n        # other info (e.g. actions) across many pages\n        for t in self.metadata['terms']:\n            if bill['session'] in t['sessions']:\n                term_year = t['start_year']\n                break\n        measureno = bill['bill_id'].replace(' ', '')\n        url = (\"http://www.arkleg.state.ar.us/assembly/%s/%s/\"\n               \"Pages/BillInformation.aspx?measureno=%s\" % (\n                   term_year, self.slug, measureno))\n        bill.add_source(url)\n\n        page = lxml.html.fromstring(self.get(url).text)\n        page.make_links_absolute(url)\n\n        for link in page.xpath(\"//a[contains(@href, 'Amendments')]\"):\n            num = link.xpath(\"string(../../td[2])\")\n            name = \"Amendment %s\" % num\n            bill.add_document(name, link.attrib['href'])\n\n        try:\n            cosponsor_link = page.xpath(\n                \"//a[contains(@href, 'CoSponsors')]\")[0]\n            self.scrape_cosponsors(bill, cosponsor_link.attrib['href'])\n        except IndexError:\n            # No cosponsor link is OK\n            pass\n\n    #     hist_link = page.xpath(\"//a[contains(@href, 'BillStatusHistory')]\")[0]\n    #     self.scrape_votes(bill, hist_link.attrib['href'])\n\n    # def scrape_votes(self, bill, url):\n    #     page = lxml.html.fromstring(self.get(url).text)\n    #     page.make_links_absolute(url)\n\n        for link in page.xpath(\"//a[contains(@href, 'votes.aspx')]\"):\n            date = link.xpath(\"string(../../td[2])\")\n            date = datetime.datetime.strptime(date, \"%m/%d/%Y %I:%M:%S %p\")\n\n            motion = link.xpath(\"string(../../td[3])\")\n\n            self.scrape_vote(bill, date, motion, link.attrib['href'])\n\n    def scrape_vote(self, bill, date, motion, url):\n        try:\n            page = self.get(url).text\n        except scrapelib.HTTPError:\n            #sometiems the link is there but is dead\n            return\n\n        if 'not yet official' in page:\n            # Sometimes they link to vote pages before they go live\n            return\n\n        page = lxml.html.fromstring(page)\n\n        if url.endswith('Senate'):\n            actor = 'upper'\n        else:\n            actor = 'lower'\n\n        count_path = \"string(//td[@align = 'center' and contains(., '%s: ')])\"\n        yes_count = int(page.xpath(count_path % \"Yeas\").split()[-1])\n        no_count = int(page.xpath(count_path % \"Nays\").split()[-1])\n        other_count = int(page.xpath(count_path % \"Non Voting\").split()[-1])\n        other_count += int(page.xpath(count_path % \"Present\").split()[-1])\n\n        passed = yes_count > no_count + other_count\n        vote = Vote(actor, date, motion, passed, yes_count,\n                    no_count, other_count)\n        vote.add_source(url)\n\n        xpath = (\n            '//*[contains(@class, \"ms-standardheader\")]/'\n            'following-sibling::table')\n        divs = page.xpath(xpath)\n        votevals = 'yes no other other'.split()\n        for (voteval, div) in zip(votevals, divs):\n            for a in div.xpath('.//a'):\n                name = a.text_content().strip()\n                if not name:\n                    continue\n                getattr(vote, voteval)(name)\n        bill.add_vote(vote)\n\n    def scrape_cosponsors(self, bill, url):\n        page = self.get(url).text\n        page = lxml.html.fromstring(page)\n", "path": "openstates/ar/bills.py"}], "after_files": [{"content": "import re\nimport csv\nimport StringIO\nimport datetime\n\nfrom billy.scrape.bills import BillScraper, Bill\nfrom billy.scrape.votes import Vote\n\nimport lxml.html\n\nimport scrapelib\n\n\ndef unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):\n    # csv.py doesn't do Unicode; encode temporarily as UTF-8:\n    csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),\n                            dialect=dialect, **kwargs)\n    for row in csv_reader:\n        # decode UTF-8 back to Unicode, cell by cell:\n        yield [unicode(cell, 'utf-8') for cell in row]\n\n\ndef utf_8_encoder(unicode_csv_data):\n    for line in unicode_csv_data:\n        yield line.encode('utf-8')\n\n\nclass ARBillScraper(BillScraper):\n    jurisdiction = 'ar'\n\n    def scrape(self, chamber, session):\n        self.bills = {}\n\n        self.slug = self.metadata['session_details'][session]['slug']\n\n        self.scrape_bill(chamber, session)\n        self.scrape_actions()\n\n        for bill in self.bills.itervalues():\n            self.save_bill(bill)\n\n    def scrape_bill(self, chamber, session):\n        url = \"ftp://www.arkleg.state.ar.us/dfadooas/LegislativeMeasures.txt\"\n        page = self.get(url).text\n        page = unicode_csv_reader(StringIO.StringIO(page), delimiter='|')\n\n        for row in page:\n            bill_chamber = {'H': 'lower', 'S': 'upper'}[row[0]]\n            if bill_chamber != chamber:\n                continue\n\n            bill_id = \"%s%s %s\" % (row[0], row[1], row[2])\n\n            type_spec = re.match(r'(H|S)([A-Z]+)\\s', bill_id).group(2)\n            bill_type = {\n                'B': 'bill',\n                'R': 'resolution',\n                'JR': 'joint resolution',\n                'CR': 'concurrent resolution',\n                'MR': 'memorial resolution',\n                'CMR': 'concurrent memorial resolution'}[type_spec]\n\n            if row[-1] != self.slug:\n                continue\n\n            bill = Bill(session, chamber, bill_id, row[3], type=bill_type)\n            bill.add_source(url)\n\n            primary = row[11]\n            if not primary:\n                primary = row[12]\n            if primary:\n                bill.add_sponsor('primary', primary)\n\n            # ftp://www.arkleg.state.ar.us/Bills/\n            # TODO: Keep on eye on this post 2017 to see if they apply R going forward.\n            session_code = '2017R' if session == '2017' else session\n\n            version_url = (\"ftp://www.arkleg.state.ar.us/Bills/\"\n                           \"%s/Public/%s.pdf\" % (\n                               session_code, bill_id.replace(' ', '')))\n            bill.add_version(bill_id, version_url, mimetype='application/pdf')\n\n            self.scrape_bill_page(bill)\n\n            self.bills[bill_id] = bill\n\n    def scrape_actions(self):\n        url = \"ftp://www.arkleg.state.ar.us/dfadooas/ChamberActions.txt\"\n        page = self.get(url).text\n        page = csv.reader(StringIO.StringIO(page))\n\n        for row in page:\n            bill_id = \"%s%s %s\" % (row[1], row[2], row[3])\n\n            if bill_id not in self.bills:\n                continue\n            # different term\n            if row[-2] != self.slug:\n                continue\n\n            # Commas aren't escaped, but only one field (the action) can\n            # contain them so we can work around it by using both positive\n            # and negative offsets\n            bill_id = \"%s%s %s\" % (row[1], row[2], row[3])\n            actor = {'HU': 'lower', 'SU': 'upper'}[row[-5].upper()]\n            # manual fix for crazy time value\n            row[6] = row[6].replace('.520000000', '')\n            date = datetime.datetime.strptime(row[6], \"%Y-%m-%d %H:%M:%S\")\n            action = ','.join(row[7:-5])\n\n            action_type = []\n            if action.startswith('Filed'):\n                action_type.append('bill:introduced')\n            elif (action.startswith('Read first time') or\n                  action.startswith('Read the first time')):\n                action_type.append('bill:reading:1')\n            if re.match('Read the first time, .*, read the second time', action):\n                action_type.append('bill:reading:2')\n            elif action.startswith('Read the third time and passed'):\n                action_type.append('bill:passed')\n                action_type.append('bill:reading:3')\n            elif action.startswith('Read the third time'):\n                action_type.append('bill:reading:3')\n            elif action.startswith('DELIVERED TO GOVERNOR'):\n                action_type.append('governor:received')\n            elif action.startswith('Notification'):\n                action_type.append('governor:signed')\n\n            if 'referred to' in action:\n                action_type.append('committee:referred')\n\n            if 'Returned by the Committee' in action:\n                if 'recommendation that it Do Pass' in action:\n                    action_type.append('committee:passed:favorable')\n                else:\n                    action_type.append('committee:passed')\n\n            if re.match(r'Amendment No\\. \\d+ read and adopted', action):\n                action_type.append('amendment:introduced')\n                action_type.append('amendment:passed')\n\n            if not action:\n                action = '[No text provided]'\n            self.bills[bill_id].add_action(actor, action, date,\n                                           type=action_type or ['other'])\n\n    def scrape_bill_page(self, bill):\n        # We need to scrape each bill page in order to grab associated votes.\n        # It's still more efficient to get the rest of the data we're\n        # interested in from the CSVs, though, because their site splits\n        # other info (e.g. actions) across many pages\n        for t in self.metadata['terms']:\n            if bill['session'] in t['sessions']:\n                term_year = t['start_year']\n                break\n        measureno = bill['bill_id'].replace(' ', '')\n        url = (\"http://www.arkleg.state.ar.us/assembly/%s/%s/\"\n               \"Pages/BillInformation.aspx?measureno=%s\" % (\n                   term_year, self.slug, measureno))\n        bill.add_source(url)\n\n        page = lxml.html.fromstring(self.get(url).text)\n        page.make_links_absolute(url)\n\n        for link in page.xpath(\"//a[contains(@href, 'Amendments')]\"):\n            num = link.xpath(\"string(../../td[2])\")\n            name = \"Amendment %s\" % num\n            bill.add_document(name, link.attrib['href'])\n\n        try:\n            cosponsor_link = page.xpath(\n                \"//a[contains(@href, 'CoSponsors')]\")[0]\n            self.scrape_cosponsors(bill, cosponsor_link.attrib['href'])\n        except IndexError:\n            # No cosponsor link is OK\n            pass\n\n    #     hist_link = page.xpath(\"//a[contains(@href, 'BillStatusHistory')]\")[0]\n    #     self.scrape_votes(bill, hist_link.attrib['href'])\n\n    # def scrape_votes(self, bill, url):\n    #     page = lxml.html.fromstring(self.get(url).text)\n    #     page.make_links_absolute(url)\n\n        for link in page.xpath(\"//a[contains(@href, 'votes.aspx')]\"):\n            date = link.xpath(\"string(../../td[2])\")\n            date = datetime.datetime.strptime(date, \"%m/%d/%Y %I:%M:%S %p\")\n\n            motion = link.xpath(\"string(../../td[3])\")\n\n            self.scrape_vote(bill, date, motion, link.attrib['href'])\n\n    def scrape_vote(self, bill, date, motion, url):\n        try:\n            page = self.get(url).text\n        except scrapelib.HTTPError:\n            #sometiems the link is there but is dead\n            return\n\n        if 'not yet official' in page:\n            # Sometimes they link to vote pages before they go live\n            return\n\n        page = lxml.html.fromstring(page)\n\n        if url.endswith('Senate'):\n            actor = 'upper'\n        else:\n            actor = 'lower'\n\n        count_path = \"string(//td[@align = 'center' and contains(., '%s: ')])\"\n        yes_count = int(page.xpath(count_path % \"Yeas\").split()[-1])\n        no_count = int(page.xpath(count_path % \"Nays\").split()[-1])\n        other_count = int(page.xpath(count_path % \"Non Voting\").split()[-1])\n        other_count += int(page.xpath(count_path % \"Present\").split()[-1])\n\n        passed = yes_count > no_count + other_count\n        vote = Vote(actor, date, motion, passed, yes_count,\n                    no_count, other_count)\n        vote.add_source(url)\n\n        xpath = (\n            '//*[contains(@class, \"ms-standardheader\")]/'\n            'following-sibling::table')\n        divs = page.xpath(xpath)\n        votevals = 'yes no other other'.split()\n        for (voteval, div) in zip(votevals, divs):\n            for a in div.xpath('.//a'):\n                name = a.text_content().strip()\n                if not name:\n                    continue\n                getattr(vote, voteval)(name)\n        bill.add_vote(vote)\n\n    def scrape_cosponsors(self, bill, url):\n        page = self.get(url).text\n        page = lxml.html.fromstring(page)\n", "path": "openstates/ar/bills.py"}]}