We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
AZ needs a new vote scraper
old bill scraper included Votes, new one doesn't

might be best to wait and do this as part of pupa version

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `openstates/az/bills.py`
Content:
```
1 import json
2 import datetime
3 
4 from pupa.scrape import Scraper, Bill
5 from . import utils
6 from . import session_metadata
7 
8 from lxml import html
9 
10 
11 BASE_URL = 'https://www.azleg.gov/'
12 
13 
14 class AZBillScraper(Scraper):
15     chamber_map = {'lower': 'H', 'upper': 'S'}
16     chamber_map_rev = {'H': 'upper', 'S': 'lower', 'G': 'executive', 'SS': 'executive'}
17     chamber_map_rev_eng = {'H': 'House', 'S': 'Senate', 'G': 'Governor',
18                            'SS': 'Secretary of State'}
19 
20     def scrape_bill(self, chamber, session, bill_id, session_id):
21         bill_json_url = 'https://apps.azleg.gov/api/Bill/?billNumber={}&sessionId={}&' \
22                         'legislativeBody={}'.format(bill_id, session_id, self.chamber_map[chamber])
23         response = self.get(bill_json_url)
24         page = json.loads(response.content.decode('utf-8'))
25 
26         if not page:
27             self.warning('null page for %s', bill_id)
28             return
29 
30         bill_title = page['ShortTitle']
31         bill_id = page['Number']
32         internal_id = page['BillId']
33         bill_type = self.get_bill_type(bill_id)
34         bill = Bill(
35             bill_id,
36             legislative_session=session,
37             chamber=chamber,
38             title=bill_title,
39             classification=bill_type,
40         )
41 
42         self.scrape_actions(bill, page, chamber)
43         self.scrape_versions_and_documents(bill, internal_id)
44         self.scrape_sponsors(bill, internal_id)
45         self.scrape_subjects(bill, internal_id)
46 
47         bill_url = 'https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}'.format(
48                     internal_id, session_id)
49         bill.add_source(bill_url)
50 
51         self.sort_bill_actions(bill)
52 
53         yield bill
54 
55     def scrape_versions_and_documents(self, bill, internal_id):
56         # Careful, this sends XML to a browser but JSON to machines
57         # https://apps.azleg.gov/api/DocType/?billStatusId=68408
58         versions_url = 'https://apps.azleg.gov/api/DocType/?billStatusId={}'.format(internal_id)
59         page = json.loads(self.get(versions_url).content.decode('utf-8'))
60         for document_set in page:
61             type_ = document_set['DocumentGroupName']
62             for doc in document_set['Documents']:
63                 media_type = 'text/html' if doc['HtmlPath'] else 'application/pdf'
64                 url = doc['HtmlPath'] or doc['PdfPath']
65                 if not url:
66                     self.warning("No PDF or HTML version found for %s" % doc['DocumentName'])
67                 # Sometimes the URL is just a relative path; make it absolute
68                 if not url.startswith('http'):
69                     url = 'https://apps.azleg.gov{}'.format(url)
70 
71                 if type_ == 'Bill Versions':
72                     bill.add_version_link(
73                         note=doc['DocumentName'],
74                         url=url,
75                         media_type=media_type
76                     )
77                 else:
78                     bill.add_document_link(
79                         note=doc['DocumentName'],
80                         url=url,
81                         media_type=media_type
82                     )
83 
84     def scrape_sponsors(self, bill, internal_id):
85         # Careful, this sends XML to a browser but JSON to machines
86         # https://apps.azleg.gov/api/BillSponsor/?id=68398
87         sponsors_url = 'https://apps.azleg.gov/api/BillSponsor/?id={}'.format(internal_id)
88         page = json.loads(self.get(sponsors_url).content.decode('utf-8'))
89         for sponsor in page:
90             if 'Prime' in sponsor['SponsorType']:
91                 sponsor_type = 'primary'
92             else:
93                 sponsor_type = 'cosponsor'
94 
95             # Some older bills don't have the FullName key
96             if 'FullName' in sponsor['Legislator']:
97                 sponsor_name = sponsor['Legislator']['FullName']
98             else:
99                 sponsor_name = "{} {}".format(
100                     sponsor['Legislator']['FirstName'],
101                     sponsor['Legislator']['LastName'],
102                 )
103             bill.add_sponsorship(
104                 classification=str(sponsor_type),
105                 name=sponsor_name,
106                 entity_type='person',
107                 primary=sponsor_type == 'primary'
108             )
109 
110     def scrape_subjects(self, bill, internal_id):
111         # https://apps.azleg.gov/api/Keyword/?billStatusId=68149
112         subjects_url = 'https://apps.azleg.gov/api/Keyword/?billStatusId={}'.format(internal_id)
113         page = json.loads(self.get(subjects_url).content.decode('utf-8'))
114         for subject in page:
115             bill.add_subject(subject['Name'])
116 
117     def scrape_actions(self, bill, page, self_chamber):
118         """
119         Scrape the actions for a given bill
120 
121         AZ No longer provides a full list, just a series of keys and dates.
122         So map that backwards using action_map
123         """
124         for status in page['BillStatusAction']:
125             if status['Action'] in utils.status_action_map:
126                 category = utils.status_action_map[status['Action']]
127                 if status['Committee']['TypeName'] == 'Floor':
128                     categories = [category]
129                     if status['Committee']['CommitteeShortName'] == 'THIRD':
130                         categories.append('reading-3')
131                 elif status['Committee']['TypeName'] == 'Standing':
132                     categories = ['committee-{}'.format(category)]
133                 else:
134                     raise ValueError(
135                         'Unexpected committee type: {}'.format(status['Committee']['TypeName']))
136                 action_date = datetime.datetime.strptime(
137                     status['ReportDate'], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')
138                 bill.add_action(
139                     description=status['Action'],
140                     chamber={
141                         'S': 'upper',
142                         'H': 'lower',
143                     }[status['Committee']['LegislativeBody']],
144                     date=action_date,
145                     classification=categories,
146                 )
147         for action in utils.action_map:
148             if page[action] and utils.action_map[action]['name'] != '':
149                 try:
150                     action_date = datetime.datetime.strptime(
151                         page[action], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')
152 
153                     bill.add_action(
154                         chamber=self.actor_from_action(bill, action, self_chamber),
155                         description=utils.action_map[action]['name'],
156                         date=action_date,
157                         classification=utils.action_map[action]['action'],
158                     )
159                 except (ValueError, TypeError):
160                     self.info("Invalid Action Time {} for {}".format(page[action], action))
161 
162         # Governor Signs and Vetos get different treatment
163         if page['GovernorAction'] == 'Signed':
164             action_date = page['GovernorActionDate'].split('T')[0]
165             bill.add_action(
166                 chamber='executive',
167                 description='Signed by Governor',
168                 date=action_date,
169                 classification='executive-signature',
170             )
171 
172         if page['GovernorAction'] == 'Vetoed':
173             action_date = page['GovernorActionDate'].split('T')[0]
174             bill.add_action(
175                 chamber='executive',
176                 description='Vetoed by Governor',
177                 date=action_date,
178                 classification='executive-veto',
179             )
180 
181         # Transmit to (X) has its own data structure as well
182         for transmit in page['BodyTransmittedTo']:
183             action_date = transmit['TransmitDate'].split('T')[0]
184             # upper, lower, executive
185             action_actor = self.chamber_map_rev[transmit['LegislativeBody']]
186             # house, senate, governor
187             body_text = self.chamber_map_rev_eng[transmit['LegislativeBody']]
188 
189             action_text = 'Transmit to {}'.format(body_text)
190 
191             if action_actor == 'executive':
192                 action_type = 'executive-receipt'
193             else:
194                 action_type = None
195 
196             bill.add_action(
197                 chamber=action_actor,
198                 description=action_text,
199                 date=action_date,
200                 classification=action_type
201             )
202 
203     def actor_from_action(self, bill, action, self_chamber):
204         """
205         Determine the actor from the action key
206         If the action_map = 'chamber', return the bill's home chamber
207         """
208         action_map = utils.action_chamber_map
209         for key in action_map:
210             if key in action:
211                 if action_map[key] == 'chamber':
212                     return self_chamber
213                 else:
214                     return action_map[key]
215 
216     def scrape(self, chamber=None, session=None):
217         if not session:
218             session = self.latest_session()
219             self.info('no session specified, using %s', session)
220         session_id = session_metadata.session_id_meta_data[session]
221 
222         # Get the bills page to start the session
223         req = self.get('https://www.azleg.gov/bills/')
224 
225         session_form_url = 'https://www.azleg.gov/azlegwp/setsession.php'
226         form = {
227             'sessionID': session_id
228         }
229         req = self.post(url=session_form_url, data=form, cookies=req.cookies, allow_redirects=True)
230 
231         bill_list_url = 'https://www.azleg.gov/bills/'
232 
233         page = self.get(bill_list_url, cookies=req.cookies).content
234         # There's an errant close-comment that browsers handle
235         # but LXML gets really confused.
236         page = page.replace(b'--!>', b'-->')
237         page = html.fromstring(page)
238 
239         bill_rows = []
240         chambers = [chamber] if chamber else ['upper', 'lower']
241         for chamber in chambers:
242             if chamber == 'lower':
243                 bill_rows = page.xpath('//div[@name="HBTable"]//tbody//tr')
244             else:
245                 bill_rows = page.xpath('//div[@name="SBTable"]//tbody//tr')
246             for row in bill_rows:
247                 bill_id = row.xpath('td/a/text()')[0]
248                 yield from self.scrape_bill(chamber, session, bill_id, session_id)
249 
250         # TODO: MBTable - Non-bill Misc Motions?
251 
252     def sort_bill_actions(self, bill):
253         actions = bill.actions
254         actions_list = []
255         out_of_order = []
256         new_list = []
257         if not actions:
258             return bill
259         action_date = actions[0]['date']
260         actions[0]['description'] = actions[0]['description'].lower()
261         actions_list.append(actions[0])
262         # seperate the actions that are out of order
263         for action in actions[1:]:
264             if action['date'] < action_date:
265                 out_of_order.append(action)
266             else:
267                 actions_list.append(action)
268                 action_date = action['date']
269             action['description'] = action['description'].lower()
270         action_date = actions_list[0]['date']
271 
272         for action in actions_list:
273             # this takes care of the actions in beween
274             for act in out_of_order:
275                 if act['date'] < action_date:
276                     o_index = out_of_order.index(act)
277                     new_list.append(out_of_order.pop(o_index))
278                 if act['date'] >= action_date and act['date'] < action['date']:
279                     o_index = out_of_order.index(act)
280                     new_list.append(out_of_order.pop(o_index))
281             new_list.append(action)
282 
283             for act in out_of_order:
284                 if act['date'] == action['date']:
285                     o_index = out_of_order.index(act)
286                     new_list.append(out_of_order.pop(o_index))
287 
288         if out_of_order != []:
289             self.info("Unable to sort " + bill.identifier)
290         else:
291             bill.actions = new_list
292 
293     def get_bill_type(self, bill_id):
294         for key in utils.bill_types:
295             if key in bill_id.lower():
296                 return utils.bill_types[key]
297         return None
298
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/openstates/az/bills.py b/openstates/az/bills.py
--- a/openstates/az/bills.py
+++ b/openstates/az/bills.py
@@ -1,12 +1,12 @@
 import json
 import datetime
 
-from pupa.scrape import Scraper, Bill
+from lxml import html
+from pupa.scrape import Scraper, Bill, VoteEvent
+
 from . import utils
 from . import session_metadata
 
-from lxml import html
-
 
 BASE_URL = 'https://www.azleg.gov/'
 
@@ -43,6 +43,7 @@
         self.scrape_versions_and_documents(bill, internal_id)
         self.scrape_sponsors(bill, internal_id)
         self.scrape_subjects(bill, internal_id)
+        yield from self.scrape_votes(bill, page)
 
         bill_url = 'https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}'.format(
                     internal_id, session_id)
@@ -213,6 +214,52 @@
                 else:
                     return action_map[key]
 
+    def scrape_votes(self, bill, page):
+        base_url = 'https://apps.azleg.gov/api/BillStatusFloorAction'
+        for header in page['FloorHeaders']:
+            params = {
+                'billStatusId': page['BillId'],
+                'billStatusActionId': header['BillStatusActionId'],
+                'includeVotes': 'true',
+            }
+            resp = self.get(base_url, params=params)
+            actions = resp.json
+            for action in actions:
+                if action['Action'] == 'No Action':
+                    continue
+                action_date = datetime.datetime.strptime(action['ReportDate'], '%Y-%m-%dT%H:%M:%S')
+                vote = VoteEvent(
+                    chamber={
+                        'S': 'upper',
+                        'H': 'lower',
+                    }[header['LegislativeBody']],
+                    motion_text=action['Action'],
+                    classification='passage',
+                    result=(
+                        'pass'
+                        if action['UnanimouslyAdopted'] or action['Ayes'] > action['Nays']
+                        else 'fail'
+                    ),
+                    start_date=action_date.strftime('%Y-%m-%d'),
+                    bill=bill,
+                )
+                vote.add_source(resp.url)
+                vote.set_count('yes', action['Ayes'] or 0)
+                vote.set_count('no', action['Nays'] or 0)
+                vote.set_count('other', (action['Present'] or 0))
+                vote.set_count('absent', (action['Absent'] or 0))
+                vote.set_count('excused', (action['Excused'] or 0))
+                vote.set_count('not voting' (action['NotVoting'] or 0))
+
+                for v in action['Votes']:
+                    vote_type = {
+                        'Y': 'yes',
+                        'N': 'no',
+                    }.get(v['Vote'], 'other')
+                    vote.vote(vote_type, v['Legislator']['FullName'])
+
+                yield vote
+
     def scrape(self, chamber=None, session=None):
         if not session:
             session = self.latest_session()

{"golden_diff": "diff --git a/openstates/az/bills.py b/openstates/az/bills.py\n--- a/openstates/az/bills.py\n+++ b/openstates/az/bills.py\n@@ -1,12 +1,12 @@\n import json\n import datetime\n \n-from pupa.scrape import Scraper, Bill\n+from lxml import html\n+from pupa.scrape import Scraper, Bill, VoteEvent\n+\n from . import utils\n from . import session_metadata\n \n-from lxml import html\n-\n \n BASE_URL = 'https://www.azleg.gov/'\n \n@@ -43,6 +43,7 @@\n         self.scrape_versions_and_documents(bill, internal_id)\n         self.scrape_sponsors(bill, internal_id)\n         self.scrape_subjects(bill, internal_id)\n+        yield from self.scrape_votes(bill, page)\n \n         bill_url = 'https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}'.format(\n                     internal_id, session_id)\n@@ -213,6 +214,52 @@\n                 else:\n                     return action_map[key]\n \n+    def scrape_votes(self, bill, page):\n+        base_url = 'https://apps.azleg.gov/api/BillStatusFloorAction'\n+        for header in page['FloorHeaders']:\n+            params = {\n+                'billStatusId': page['BillId'],\n+                'billStatusActionId': header['BillStatusActionId'],\n+                'includeVotes': 'true',\n+            }\n+            resp = self.get(base_url, params=params)\n+            actions = resp.json\n+            for action in actions:\n+                if action['Action'] == 'No Action':\n+                    continue\n+                action_date = datetime.datetime.strptime(action['ReportDate'], '%Y-%m-%dT%H:%M:%S')\n+                vote = VoteEvent(\n+                    chamber={\n+                        'S': 'upper',\n+                        'H': 'lower',\n+                    }[header['LegislativeBody']],\n+                    motion_text=action['Action'],\n+                    classification='passage',\n+                    result=(\n+                        'pass'\n+                        if action['UnanimouslyAdopted'] or action['Ayes'] > action['Nays']\n+                        else 'fail'\n+                    ),\n+                    start_date=action_date.strftime('%Y-%m-%d'),\n+                    bill=bill,\n+                )\n+                vote.add_source(resp.url)\n+                vote.set_count('yes', action['Ayes'] or 0)\n+                vote.set_count('no', action['Nays'] or 0)\n+                vote.set_count('other', (action['Present'] or 0))\n+                vote.set_count('absent', (action['Absent'] or 0))\n+                vote.set_count('excused', (action['Excused'] or 0))\n+                vote.set_count('not voting' (action['NotVoting'] or 0))\n+\n+                for v in action['Votes']:\n+                    vote_type = {\n+                        'Y': 'yes',\n+                        'N': 'no',\n+                    }.get(v['Vote'], 'other')\n+                    vote.vote(vote_type, v['Legislator']['FullName'])\n+\n+                yield vote\n+\n     def scrape(self, chamber=None, session=None):\n         if not session:\n             session = self.latest_session()\n", "issue": "AZ needs a new vote scraper\nold bill scraper included Votes, new one doesn't\n\nmight be best to wait and do this as part of pupa version\n", "before_files": [{"content": "import json\nimport datetime\n\nfrom pupa.scrape import Scraper, Bill\nfrom . import utils\nfrom . import session_metadata\n\nfrom lxml import html\n\n\nBASE_URL = 'https://www.azleg.gov/'\n\n\nclass AZBillScraper(Scraper):\n    chamber_map = {'lower': 'H', 'upper': 'S'}\n    chamber_map_rev = {'H': 'upper', 'S': 'lower', 'G': 'executive', 'SS': 'executive'}\n    chamber_map_rev_eng = {'H': 'House', 'S': 'Senate', 'G': 'Governor',\n                           'SS': 'Secretary of State'}\n\n    def scrape_bill(self, chamber, session, bill_id, session_id):\n        bill_json_url = 'https://apps.azleg.gov/api/Bill/?billNumber={}&sessionId={}&' \\\n                        'legislativeBody={}'.format(bill_id, session_id, self.chamber_map[chamber])\n        response = self.get(bill_json_url)\n        page = json.loads(response.content.decode('utf-8'))\n\n        if not page:\n            self.warning('null page for %s', bill_id)\n            return\n\n        bill_title = page['ShortTitle']\n        bill_id = page['Number']\n        internal_id = page['BillId']\n        bill_type = self.get_bill_type(bill_id)\n        bill = Bill(\n            bill_id,\n            legislative_session=session,\n            chamber=chamber,\n            title=bill_title,\n            classification=bill_type,\n        )\n\n        self.scrape_actions(bill, page, chamber)\n        self.scrape_versions_and_documents(bill, internal_id)\n        self.scrape_sponsors(bill, internal_id)\n        self.scrape_subjects(bill, internal_id)\n\n        bill_url = 'https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}'.format(\n                    internal_id, session_id)\n        bill.add_source(bill_url)\n\n        self.sort_bill_actions(bill)\n\n        yield bill\n\n    def scrape_versions_and_documents(self, bill, internal_id):\n        # Careful, this sends XML to a browser but JSON to machines\n        # https://apps.azleg.gov/api/DocType/?billStatusId=68408\n        versions_url = 'https://apps.azleg.gov/api/DocType/?billStatusId={}'.format(internal_id)\n        page = json.loads(self.get(versions_url).content.decode('utf-8'))\n        for document_set in page:\n            type_ = document_set['DocumentGroupName']\n            for doc in document_set['Documents']:\n                media_type = 'text/html' if doc['HtmlPath'] else 'application/pdf'\n                url = doc['HtmlPath'] or doc['PdfPath']\n                if not url:\n                    self.warning(\"No PDF or HTML version found for %s\" % doc['DocumentName'])\n                # Sometimes the URL is just a relative path; make it absolute\n                if not url.startswith('http'):\n                    url = 'https://apps.azleg.gov{}'.format(url)\n\n                if type_ == 'Bill Versions':\n                    bill.add_version_link(\n                        note=doc['DocumentName'],\n                        url=url,\n                        media_type=media_type\n                    )\n                else:\n                    bill.add_document_link(\n                        note=doc['DocumentName'],\n                        url=url,\n                        media_type=media_type\n                    )\n\n    def scrape_sponsors(self, bill, internal_id):\n        # Careful, this sends XML to a browser but JSON to machines\n        # https://apps.azleg.gov/api/BillSponsor/?id=68398\n        sponsors_url = 'https://apps.azleg.gov/api/BillSponsor/?id={}'.format(internal_id)\n        page = json.loads(self.get(sponsors_url).content.decode('utf-8'))\n        for sponsor in page:\n            if 'Prime' in sponsor['SponsorType']:\n                sponsor_type = 'primary'\n            else:\n                sponsor_type = 'cosponsor'\n\n            # Some older bills don't have the FullName key\n            if 'FullName' in sponsor['Legislator']:\n                sponsor_name = sponsor['Legislator']['FullName']\n            else:\n                sponsor_name = \"{} {}\".format(\n                    sponsor['Legislator']['FirstName'],\n                    sponsor['Legislator']['LastName'],\n                )\n            bill.add_sponsorship(\n                classification=str(sponsor_type),\n                name=sponsor_name,\n                entity_type='person',\n                primary=sponsor_type == 'primary'\n            )\n\n    def scrape_subjects(self, bill, internal_id):\n        # https://apps.azleg.gov/api/Keyword/?billStatusId=68149\n        subjects_url = 'https://apps.azleg.gov/api/Keyword/?billStatusId={}'.format(internal_id)\n        page = json.loads(self.get(subjects_url).content.decode('utf-8'))\n        for subject in page:\n            bill.add_subject(subject['Name'])\n\n    def scrape_actions(self, bill, page, self_chamber):\n        \"\"\"\n        Scrape the actions for a given bill\n\n        AZ No longer provides a full list, just a series of keys and dates.\n        So map that backwards using action_map\n        \"\"\"\n        for status in page['BillStatusAction']:\n            if status['Action'] in utils.status_action_map:\n                category = utils.status_action_map[status['Action']]\n                if status['Committee']['TypeName'] == 'Floor':\n                    categories = [category]\n                    if status['Committee']['CommitteeShortName'] == 'THIRD':\n                        categories.append('reading-3')\n                elif status['Committee']['TypeName'] == 'Standing':\n                    categories = ['committee-{}'.format(category)]\n                else:\n                    raise ValueError(\n                        'Unexpected committee type: {}'.format(status['Committee']['TypeName']))\n                action_date = datetime.datetime.strptime(\n                    status['ReportDate'], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')\n                bill.add_action(\n                    description=status['Action'],\n                    chamber={\n                        'S': 'upper',\n                        'H': 'lower',\n                    }[status['Committee']['LegislativeBody']],\n                    date=action_date,\n                    classification=categories,\n                )\n        for action in utils.action_map:\n            if page[action] and utils.action_map[action]['name'] != '':\n                try:\n                    action_date = datetime.datetime.strptime(\n                        page[action], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')\n\n                    bill.add_action(\n                        chamber=self.actor_from_action(bill, action, self_chamber),\n                        description=utils.action_map[action]['name'],\n                        date=action_date,\n                        classification=utils.action_map[action]['action'],\n                    )\n                except (ValueError, TypeError):\n                    self.info(\"Invalid Action Time {} for {}\".format(page[action], action))\n\n        # Governor Signs and Vetos get different treatment\n        if page['GovernorAction'] == 'Signed':\n            action_date = page['GovernorActionDate'].split('T')[0]\n            bill.add_action(\n                chamber='executive',\n                description='Signed by Governor',\n                date=action_date,\n                classification='executive-signature',\n            )\n\n        if page['GovernorAction'] == 'Vetoed':\n            action_date = page['GovernorActionDate'].split('T')[0]\n            bill.add_action(\n                chamber='executive',\n                description='Vetoed by Governor',\n                date=action_date,\n                classification='executive-veto',\n            )\n\n        # Transmit to (X) has its own data structure as well\n        for transmit in page['BodyTransmittedTo']:\n            action_date = transmit['TransmitDate'].split('T')[0]\n            # upper, lower, executive\n            action_actor = self.chamber_map_rev[transmit['LegislativeBody']]\n            # house, senate, governor\n            body_text = self.chamber_map_rev_eng[transmit['LegislativeBody']]\n\n            action_text = 'Transmit to {}'.format(body_text)\n\n            if action_actor == 'executive':\n                action_type = 'executive-receipt'\n            else:\n                action_type = None\n\n            bill.add_action(\n                chamber=action_actor,\n                description=action_text,\n                date=action_date,\n                classification=action_type\n            )\n\n    def actor_from_action(self, bill, action, self_chamber):\n        \"\"\"\n        Determine the actor from the action key\n        If the action_map = 'chamber', return the bill's home chamber\n        \"\"\"\n        action_map = utils.action_chamber_map\n        for key in action_map:\n            if key in action:\n                if action_map[key] == 'chamber':\n                    return self_chamber\n                else:\n                    return action_map[key]\n\n    def scrape(self, chamber=None, session=None):\n        if not session:\n            session = self.latest_session()\n            self.info('no session specified, using %s', session)\n        session_id = session_metadata.session_id_meta_data[session]\n\n        # Get the bills page to start the session\n        req = self.get('https://www.azleg.gov/bills/')\n\n        session_form_url = 'https://www.azleg.gov/azlegwp/setsession.php'\n        form = {\n            'sessionID': session_id\n        }\n        req = self.post(url=session_form_url, data=form, cookies=req.cookies, allow_redirects=True)\n\n        bill_list_url = 'https://www.azleg.gov/bills/'\n\n        page = self.get(bill_list_url, cookies=req.cookies).content\n        # There's an errant close-comment that browsers handle\n        # but LXML gets really confused.\n        page = page.replace(b'--!>', b'-->')\n        page = html.fromstring(page)\n\n        bill_rows = []\n        chambers = [chamber] if chamber else ['upper', 'lower']\n        for chamber in chambers:\n            if chamber == 'lower':\n                bill_rows = page.xpath('//div[@name=\"HBTable\"]//tbody//tr')\n            else:\n                bill_rows = page.xpath('//div[@name=\"SBTable\"]//tbody//tr')\n            for row in bill_rows:\n                bill_id = row.xpath('td/a/text()')[0]\n                yield from self.scrape_bill(chamber, session, bill_id, session_id)\n\n        # TODO: MBTable - Non-bill Misc Motions?\n\n    def sort_bill_actions(self, bill):\n        actions = bill.actions\n        actions_list = []\n        out_of_order = []\n        new_list = []\n        if not actions:\n            return bill\n        action_date = actions[0]['date']\n        actions[0]['description'] = actions[0]['description'].lower()\n        actions_list.append(actions[0])\n        # seperate the actions that are out of order\n        for action in actions[1:]:\n            if action['date'] < action_date:\n                out_of_order.append(action)\n            else:\n                actions_list.append(action)\n                action_date = action['date']\n            action['description'] = action['description'].lower()\n        action_date = actions_list[0]['date']\n\n        for action in actions_list:\n            # this takes care of the actions in beween\n            for act in out_of_order:\n                if act['date'] < action_date:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n                if act['date'] >= action_date and act['date'] < action['date']:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n            new_list.append(action)\n\n            for act in out_of_order:\n                if act['date'] == action['date']:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n\n        if out_of_order != []:\n            self.info(\"Unable to sort \" + bill.identifier)\n        else:\n            bill.actions = new_list\n\n    def get_bill_type(self, bill_id):\n        for key in utils.bill_types:\n            if key in bill_id.lower():\n                return utils.bill_types[key]\n        return None\n", "path": "openstates/az/bills.py"}], "after_files": [{"content": "import json\nimport datetime\n\nfrom lxml import html\nfrom pupa.scrape import Scraper, Bill, VoteEvent\n\nfrom . import utils\nfrom . import session_metadata\n\n\nBASE_URL = 'https://www.azleg.gov/'\n\n\nclass AZBillScraper(Scraper):\n    chamber_map = {'lower': 'H', 'upper': 'S'}\n    chamber_map_rev = {'H': 'upper', 'S': 'lower', 'G': 'executive', 'SS': 'executive'}\n    chamber_map_rev_eng = {'H': 'House', 'S': 'Senate', 'G': 'Governor',\n                           'SS': 'Secretary of State'}\n\n    def scrape_bill(self, chamber, session, bill_id, session_id):\n        bill_json_url = 'https://apps.azleg.gov/api/Bill/?billNumber={}&sessionId={}&' \\\n                        'legislativeBody={}'.format(bill_id, session_id, self.chamber_map[chamber])\n        response = self.get(bill_json_url)\n        page = json.loads(response.content.decode('utf-8'))\n\n        if not page:\n            self.warning('null page for %s', bill_id)\n            return\n\n        bill_title = page['ShortTitle']\n        bill_id = page['Number']\n        internal_id = page['BillId']\n        bill_type = self.get_bill_type(bill_id)\n        bill = Bill(\n            bill_id,\n            legislative_session=session,\n            chamber=chamber,\n            title=bill_title,\n            classification=bill_type,\n        )\n\n        self.scrape_actions(bill, page, chamber)\n        self.scrape_versions_and_documents(bill, internal_id)\n        self.scrape_sponsors(bill, internal_id)\n        self.scrape_subjects(bill, internal_id)\n        yield from self.scrape_votes(bill, page)\n\n        bill_url = 'https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}'.format(\n                    internal_id, session_id)\n        bill.add_source(bill_url)\n\n        self.sort_bill_actions(bill)\n\n        yield bill\n\n    def scrape_versions_and_documents(self, bill, internal_id):\n        # Careful, this sends XML to a browser but JSON to machines\n        # https://apps.azleg.gov/api/DocType/?billStatusId=68408\n        versions_url = 'https://apps.azleg.gov/api/DocType/?billStatusId={}'.format(internal_id)\n        page = json.loads(self.get(versions_url).content.decode('utf-8'))\n        for document_set in page:\n            type_ = document_set['DocumentGroupName']\n            for doc in document_set['Documents']:\n                media_type = 'text/html' if doc['HtmlPath'] else 'application/pdf'\n                url = doc['HtmlPath'] or doc['PdfPath']\n                if not url:\n                    self.warning(\"No PDF or HTML version found for %s\" % doc['DocumentName'])\n                # Sometimes the URL is just a relative path; make it absolute\n                if not url.startswith('http'):\n                    url = 'https://apps.azleg.gov{}'.format(url)\n\n                if type_ == 'Bill Versions':\n                    bill.add_version_link(\n                        note=doc['DocumentName'],\n                        url=url,\n                        media_type=media_type\n                    )\n                else:\n                    bill.add_document_link(\n                        note=doc['DocumentName'],\n                        url=url,\n                        media_type=media_type\n                    )\n\n    def scrape_sponsors(self, bill, internal_id):\n        # Careful, this sends XML to a browser but JSON to machines\n        # https://apps.azleg.gov/api/BillSponsor/?id=68398\n        sponsors_url = 'https://apps.azleg.gov/api/BillSponsor/?id={}'.format(internal_id)\n        page = json.loads(self.get(sponsors_url).content.decode('utf-8'))\n        for sponsor in page:\n            if 'Prime' in sponsor['SponsorType']:\n                sponsor_type = 'primary'\n            else:\n                sponsor_type = 'cosponsor'\n\n            # Some older bills don't have the FullName key\n            if 'FullName' in sponsor['Legislator']:\n                sponsor_name = sponsor['Legislator']['FullName']\n            else:\n                sponsor_name = \"{} {}\".format(\n                    sponsor['Legislator']['FirstName'],\n                    sponsor['Legislator']['LastName'],\n                )\n            bill.add_sponsorship(\n                classification=str(sponsor_type),\n                name=sponsor_name,\n                entity_type='person',\n                primary=sponsor_type == 'primary'\n            )\n\n    def scrape_subjects(self, bill, internal_id):\n        # https://apps.azleg.gov/api/Keyword/?billStatusId=68149\n        subjects_url = 'https://apps.azleg.gov/api/Keyword/?billStatusId={}'.format(internal_id)\n        page = json.loads(self.get(subjects_url).content.decode('utf-8'))\n        for subject in page:\n            bill.add_subject(subject['Name'])\n\n    def scrape_actions(self, bill, page, self_chamber):\n        \"\"\"\n        Scrape the actions for a given bill\n\n        AZ No longer provides a full list, just a series of keys and dates.\n        So map that backwards using action_map\n        \"\"\"\n        for status in page['BillStatusAction']:\n            if status['Action'] in utils.status_action_map:\n                category = utils.status_action_map[status['Action']]\n                if status['Committee']['TypeName'] == 'Floor':\n                    categories = [category]\n                    if status['Committee']['CommitteeShortName'] == 'THIRD':\n                        categories.append('reading-3')\n                elif status['Committee']['TypeName'] == 'Standing':\n                    categories = ['committee-{}'.format(category)]\n                else:\n                    raise ValueError(\n                        'Unexpected committee type: {}'.format(status['Committee']['TypeName']))\n                action_date = datetime.datetime.strptime(\n                    status['ReportDate'], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')\n                bill.add_action(\n                    description=status['Action'],\n                    chamber={\n                        'S': 'upper',\n                        'H': 'lower',\n                    }[status['Committee']['LegislativeBody']],\n                    date=action_date,\n                    classification=categories,\n                )\n        for action in utils.action_map:\n            if page[action] and utils.action_map[action]['name'] != '':\n                try:\n                    action_date = datetime.datetime.strptime(\n                        page[action], '%Y-%m-%dT%H:%M:%S').strftime('%Y-%m-%d')\n\n                    bill.add_action(\n                        chamber=self.actor_from_action(bill, action, self_chamber),\n                        description=utils.action_map[action]['name'],\n                        date=action_date,\n                        classification=utils.action_map[action]['action'],\n                    )\n                except (ValueError, TypeError):\n                    self.info(\"Invalid Action Time {} for {}\".format(page[action], action))\n\n        # Governor Signs and Vetos get different treatment\n        if page['GovernorAction'] == 'Signed':\n            action_date = page['GovernorActionDate'].split('T')[0]\n            bill.add_action(\n                chamber='executive',\n                description='Signed by Governor',\n                date=action_date,\n                classification='executive-signature',\n            )\n\n        if page['GovernorAction'] == 'Vetoed':\n            action_date = page['GovernorActionDate'].split('T')[0]\n            bill.add_action(\n                chamber='executive',\n                description='Vetoed by Governor',\n                date=action_date,\n                classification='executive-veto',\n            )\n\n        # Transmit to (X) has its own data structure as well\n        for transmit in page['BodyTransmittedTo']:\n            action_date = transmit['TransmitDate'].split('T')[0]\n            # upper, lower, executive\n            action_actor = self.chamber_map_rev[transmit['LegislativeBody']]\n            # house, senate, governor\n            body_text = self.chamber_map_rev_eng[transmit['LegislativeBody']]\n\n            action_text = 'Transmit to {}'.format(body_text)\n\n            if action_actor == 'executive':\n                action_type = 'executive-receipt'\n            else:\n                action_type = None\n\n            bill.add_action(\n                chamber=action_actor,\n                description=action_text,\n                date=action_date,\n                classification=action_type\n            )\n\n    def actor_from_action(self, bill, action, self_chamber):\n        \"\"\"\n        Determine the actor from the action key\n        If the action_map = 'chamber', return the bill's home chamber\n        \"\"\"\n        action_map = utils.action_chamber_map\n        for key in action_map:\n            if key in action:\n                if action_map[key] == 'chamber':\n                    return self_chamber\n                else:\n                    return action_map[key]\n\n    def scrape_votes(self, bill, page):\n        base_url = 'https://apps.azleg.gov/api/BillStatusFloorAction'\n        for header in page['FloorHeaders']:\n            params = {\n                'billStatusId': page['BillId'],\n                'billStatusActionId': header['BillStatusActionId'],\n                'includeVotes': 'true',\n            }\n            resp = self.get(base_url, params=params)\n            actions = resp.json\n            for action in actions:\n                if action['Action'] == 'No Action':\n                    continue\n                action_date = datetime.datetime.strptime(action['ReportDate'], '%Y-%m-%dT%H:%M:%S')\n                vote = VoteEvent(\n                    chamber={\n                        'S': 'upper',\n                        'H': 'lower',\n                    }[header['LegislativeBody']],\n                    motion_text=action['Action'],\n                    classification='passage',\n                    result=(\n                        'pass'\n                        if action['UnanimouslyAdopted'] or action['Ayes'] > action['Nays']\n                        else 'fail'\n                    ),\n                    start_date=action_date.strftime('%Y-%m-%d'),\n                    bill=bill,\n                )\n                vote.add_source(resp.url)\n                vote.set_count('yes', action['Ayes'] or 0)\n                vote.set_count('no', action['Nays'] or 0)\n                vote.set_count('other', (action['Present'] or 0))\n                vote.set_count('absent', (action['Absent'] or 0))\n                vote.set_count('excused', (action['Excused'] or 0))\n                vote.set_count('not voting' (action['NotVoting'] or 0))\n\n                for v in action['Votes']:\n                    vote_type = {\n                        'Y': 'yes',\n                        'N': 'no',\n                    }.get(v['Vote'], 'other')\n                    vote.vote(vote_type, v['Legislator']['FullName'])\n\n                yield vote\n\n    def scrape(self, chamber=None, session=None):\n        if not session:\n            session = self.latest_session()\n            self.info('no session specified, using %s', session)\n        session_id = session_metadata.session_id_meta_data[session]\n\n        # Get the bills page to start the session\n        req = self.get('https://www.azleg.gov/bills/')\n\n        session_form_url = 'https://www.azleg.gov/azlegwp/setsession.php'\n        form = {\n            'sessionID': session_id\n        }\n        req = self.post(url=session_form_url, data=form, cookies=req.cookies, allow_redirects=True)\n\n        bill_list_url = 'https://www.azleg.gov/bills/'\n\n        page = self.get(bill_list_url, cookies=req.cookies).content\n        # There's an errant close-comment that browsers handle\n        # but LXML gets really confused.\n        page = page.replace(b'--!>', b'-->')\n        page = html.fromstring(page)\n\n        bill_rows = []\n        chambers = [chamber] if chamber else ['upper', 'lower']\n        for chamber in chambers:\n            if chamber == 'lower':\n                bill_rows = page.xpath('//div[@name=\"HBTable\"]//tbody//tr')\n            else:\n                bill_rows = page.xpath('//div[@name=\"SBTable\"]//tbody//tr')\n            for row in bill_rows:\n                bill_id = row.xpath('td/a/text()')[0]\n                yield from self.scrape_bill(chamber, session, bill_id, session_id)\n\n        # TODO: MBTable - Non-bill Misc Motions?\n\n    def sort_bill_actions(self, bill):\n        actions = bill.actions\n        actions_list = []\n        out_of_order = []\n        new_list = []\n        if not actions:\n            return bill\n        action_date = actions[0]['date']\n        actions[0]['description'] = actions[0]['description'].lower()\n        actions_list.append(actions[0])\n        # seperate the actions that are out of order\n        for action in actions[1:]:\n            if action['date'] < action_date:\n                out_of_order.append(action)\n            else:\n                actions_list.append(action)\n                action_date = action['date']\n            action['description'] = action['description'].lower()\n        action_date = actions_list[0]['date']\n\n        for action in actions_list:\n            # this takes care of the actions in beween\n            for act in out_of_order:\n                if act['date'] < action_date:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n                if act['date'] >= action_date and act['date'] < action['date']:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n            new_list.append(action)\n\n            for act in out_of_order:\n                if act['date'] == action['date']:\n                    o_index = out_of_order.index(act)\n                    new_list.append(out_of_order.pop(o_index))\n\n        if out_of_order != []:\n            self.info(\"Unable to sort \" + bill.identifier)\n        else:\n            bill.actions = new_list\n\n    def get_bill_type(self, bill_id):\n        for key in utils.bill_types:\n            if key in bill_id.lower():\n                return utils.bill_types[key]\n        return None\n", "path": "openstates/az/bills.py"}]}