We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
GitHub source step fails on missing magic `/merge` ref of GitHub
Currently I run into an issue where the changehook is too fast for the magic `merge` ref in GitHub to be created on Pull Requests.

An easy fix would be to try again in this case or wait a reasonable amount in `startVC()` of the GitHub step if the remote ref ends in `merge`.

Thoughts?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `master/buildbot/www/hooks/github.py`
Content:
```
1 # This file is part of Buildbot.  Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 from __future__ import absolute_import
17 from __future__ import print_function
18 
19 import hmac
20 import json
21 import logging
22 import re
23 from hashlib import sha1
24 
25 from dateutil.parser import parse as dateparse
26 
27 from twisted.internet import defer
28 from twisted.python import log
29 
30 from buildbot.changes.github import PullRequestMixin
31 from buildbot.util import bytes2unicode
32 from buildbot.util import httpclientservice
33 from buildbot.util import unicode2bytes
34 from buildbot.www.hooks.base import BaseHookHandler
35 
36 _HEADER_EVENT = b'X-GitHub-Event'
37 _HEADER_SIGNATURE = b'X-Hub-Signature'
38 
39 DEFAULT_SKIPS_PATTERN = (r'\[ *skip *ci *\]', r'\[ *ci *skip *\]')
40 DEFAULT_GITHUB_API_URL = 'https://api.github.com'
41 
42 
43 class GitHubEventHandler(PullRequestMixin):
44 
45     def __init__(self, secret, strict,
46                  codebase=None,
47                  github_property_whitelist=None,
48                  master=None,
49                  skips=None,
50                  github_api_endpoint=None,
51                  token=None,
52                  debug=False,
53                  verify=False):
54         self._secret = secret
55         self._strict = strict
56         self._token = token
57         self._codebase = codebase
58         self.github_property_whitelist = github_property_whitelist
59         self.skips = skips
60         self.github_api_endpoint = github_api_endpoint
61         self.master = master
62         if github_property_whitelist is None:
63             self.github_property_whitelist = []
64         if skips is None:
65             self.skips = DEFAULT_SKIPS_PATTERN
66         if github_api_endpoint is None:
67             self.github_api_endpoint = DEFAULT_GITHUB_API_URL
68 
69         if self._strict and not self._secret:
70             raise ValueError('Strict mode is requested '
71                              'while no secret is provided')
72         self.debug = debug
73         self.verify = verify
74 
75     @defer.inlineCallbacks
76     def process(self, request):
77         payload = self._get_payload(request)
78 
79         event_type = request.getHeader(_HEADER_EVENT)
80         event_type = bytes2unicode(event_type)
81         log.msg("X-GitHub-Event: {}".format(
82             event_type), logLevel=logging.DEBUG)
83 
84         handler = getattr(self, 'handle_{}'.format(event_type), None)
85 
86         if handler is None:
87             raise ValueError('Unknown event: {}'.format(event_type))
88 
89         result = yield defer.maybeDeferred(lambda: handler(payload, event_type))
90         defer.returnValue(result)
91 
92     def _get_payload(self, request):
93         content = request.content.read()
94         content = bytes2unicode(content)
95 
96         signature = request.getHeader(_HEADER_SIGNATURE)
97         signature = bytes2unicode(signature)
98 
99         if not signature and self._strict:
100             raise ValueError('Request has no required signature')
101 
102         if self._secret and signature:
103             try:
104                 hash_type, hexdigest = signature.split('=')
105             except ValueError:
106                 raise ValueError(
107                     'Wrong signature format: {}'.format(signature))
108 
109             if hash_type != 'sha1':
110                 raise ValueError('Unknown hash type: {}'.format(hash_type))
111 
112             mac = hmac.new(unicode2bytes(self._secret),
113                            msg=unicode2bytes(content),
114                            digestmod=sha1)
115             # NOTE: hmac.compare_digest should be used, but it's only available
116             # starting Python 2.7.7
117             if mac.hexdigest() != hexdigest:
118                 raise ValueError('Hash mismatch')
119 
120         content_type = request.getHeader(b'Content-Type')
121 
122         if content_type == b'application/json':
123             payload = json.loads(content)
124         elif content_type == b'application/x-www-form-urlencoded':
125             payload = json.loads(bytes2unicode(request.args[b'payload'][0]))
126         else:
127             raise ValueError('Unknown content type: {}'.format(content_type))
128 
129         log.msg("Payload: {}".format(payload), logLevel=logging.DEBUG)
130 
131         return payload
132 
133     def handle_ping(self, _, __):
134         return [], 'git'
135 
136     def handle_push(self, payload, event):
137         # This field is unused:
138         user = None
139         # user = payload['pusher']['name']
140         repo = payload['repository']['name']
141         repo_url = payload['repository']['html_url']
142         # NOTE: what would be a reasonable value for project?
143         # project = request.args.get('project', [''])[0]
144         project = payload['repository']['full_name']
145 
146         # Inject some additional white-listed event payload properties
147         properties = self.extractProperties(payload)
148         changes = self._process_change(payload, user, repo, repo_url, project,
149                                        event, properties)
150 
151         log.msg("Received {} changes from github".format(len(changes)))
152 
153         return changes, 'git'
154 
155     @defer.inlineCallbacks
156     def handle_pull_request(self, payload, event):
157         changes = []
158         number = payload['number']
159         refname = 'refs/pull/{}/merge'.format(number)
160         commits = payload['pull_request']['commits']
161         title = payload['pull_request']['title']
162         comments = payload['pull_request']['body']
163         repo_full_name = payload['repository']['full_name']
164         head_sha = payload['pull_request']['head']['sha']
165 
166         log.msg('Processing GitHub PR #{}'.format(number),
167                 logLevel=logging.DEBUG)
168 
169         head_msg = yield self._get_commit_msg(repo_full_name, head_sha)
170         if self._has_skip(head_msg):
171             log.msg("GitHub PR #{}, Ignoring: "
172                     "head commit message contains skip pattern".format(number))
173             defer.returnValue(([], 'git'))
174 
175         action = payload.get('action')
176         if action not in ('opened', 'reopened', 'synchronize'):
177             log.msg("GitHub PR #{} {}, ignoring".format(number, action))
178             defer.returnValue((changes, 'git'))
179 
180         properties = self.extractProperties(payload['pull_request'])
181         properties.update({'event': event})
182         change = {
183             'revision': payload['pull_request']['head']['sha'],
184             'when_timestamp': dateparse(payload['pull_request']['created_at']),
185             'branch': refname,
186             'revlink': payload['pull_request']['_links']['html']['href'],
187             'repository': payload['repository']['html_url'],
188             'project': payload['pull_request']['base']['repo']['full_name'],
189             'category': 'pull',
190             # TODO: Get author name based on login id using txgithub module
191             'author': payload['sender']['login'],
192             'comments': u'GitHub Pull Request #{0} ({1} commit{2})\n{3}\n{4}'.format(
193                 number, commits, 's' if commits != 1 else '', title, comments),
194             'properties': properties,
195         }
196 
197         if callable(self._codebase):
198             change['codebase'] = self._codebase(payload)
199         elif self._codebase is not None:
200             change['codebase'] = self._codebase
201 
202         changes.append(change)
203 
204         log.msg("Received {} changes from GitHub PR #{}".format(
205             len(changes), number))
206         defer.returnValue((changes, 'git'))
207 
208     @defer.inlineCallbacks
209     def _get_commit_msg(self, repo, sha):
210         '''
211         :param repo: the repo full name, ``{owner}/{project}``.
212             e.g. ``buildbot/buildbot``
213         '''
214         headers = {
215             'User-Agent': 'Buildbot'
216         }
217         if self._token:
218             headers['Authorization'] = 'token ' + self._token
219 
220         url = '/repos/{}/commits/{}'.format(repo, sha)
221         http = yield httpclientservice.HTTPClientService.getService(
222             self.master, self.github_api_endpoint, headers=headers,
223             debug=self.debug, verify=self.verify)
224         res = yield http.get(url)
225         data = yield res.json()
226         msg = data['commit']['message']
227         defer.returnValue(msg)
228 
229     def _process_change(self, payload, user, repo, repo_url, project, event,
230                         properties):
231         """
232         Consumes the JSON as a python object and actually starts the build.
233 
234         :arguments:
235             payload
236                 Python Object that represents the JSON sent by GitHub Service
237                 Hook.
238         """
239         changes = []
240         refname = payload['ref']
241 
242         # We only care about regular heads or tags
243         match = re.match(r"^refs/(heads|tags)/(.+)$", refname)
244         if not match:
245             log.msg("Ignoring refname `{}': Not a branch".format(refname))
246             return changes
247         category = None  # None is the legacy category for when hook only supported push
248         if match.group(1) == "tags":
249             category = "tag"
250 
251         branch = match.group(2)
252         if payload.get('deleted'):
253             log.msg("Branch `{}' deleted, ignoring".format(branch))
254             return changes
255 
256         # check skip pattern in commit message. e.g.: [ci skip] and [skip ci]
257         head_msg = payload['head_commit'].get('message', '')
258         if self._has_skip(head_msg):
259             return changes
260         commits = payload['commits']
261         if payload.get('created'):
262             commits = [payload['head_commit']]
263         for commit in commits:
264             files = []
265             for kind in ('added', 'modified', 'removed'):
266                 files.extend(commit.get(kind, []))
267 
268             when_timestamp = dateparse(commit['timestamp'])
269 
270             log.msg("New revision: {}".format(commit['id'][:8]))
271 
272             change = {
273                 'author': u'{} <{}>'.format(commit['author']['name'],
274                                            commit['author']['email']),
275                 'files': files,
276                 'comments': commit['message'],
277                 'revision': commit['id'],
278                 'when_timestamp': when_timestamp,
279                 'branch': branch,
280                 'revlink': commit['url'],
281                 'repository': repo_url,
282                 'project': project,
283                 'properties': {
284                     'github_distinct': commit.get('distinct', True),
285                     'event': event,
286                 },
287                 'category': category
288             }
289             # Update with any white-listed github event properties
290             change['properties'].update(properties)
291 
292             if callable(self._codebase):
293                 change['codebase'] = self._codebase(payload)
294             elif self._codebase is not None:
295                 change['codebase'] = self._codebase
296 
297             changes.append(change)
298 
299         return changes
300 
301     def _has_skip(self, msg):
302         '''
303         The message contains the skipping keyword no not.
304 
305         :return type: Bool
306         '''
307         for skip in self.skips:
308             if re.search(skip, msg):
309                 return True
310         return False
311 
312 # for GitHub, we do another level of indirection because
313 # we already had documented API that encouraged people to subclass GitHubEventHandler
314 # so we need to be careful not breaking that API.
315 
316 
317 class GitHubHandler(BaseHookHandler):
318     def __init__(self, master, options):
319         if options is None:
320             options = {}
321         BaseHookHandler.__init__(self, master, options)
322 
323         klass = options.get('class', GitHubEventHandler)
324         klass_kwargs = {
325             'master': master,
326             'codebase': options.get('codebase', None),
327             'github_property_whitelist': options.get('github_property_whitelist', None),
328             'skips': options.get('skips', None),
329             'github_api_endpoint': options.get('github_api_endpoint', None) or 'https://api.github.com',
330             'token': options.get('token', None),
331             'debug': options.get('debug', None) or False,
332             'verify': options.get('verify', None) or False,
333         }
334         handler = klass(options.get('secret', None),
335                         options.get('strict', False),
336                         **klass_kwargs)
337         self.handler = handler
338 
339     def getChanges(self, request):
340         return self.handler.process(request)
341 
342 
343 github = GitHubHandler
344
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/master/buildbot/www/hooks/github.py b/master/buildbot/www/hooks/github.py
--- a/master/buildbot/www/hooks/github.py
+++ b/master/buildbot/www/hooks/github.py
@@ -48,6 +48,7 @@
                  master=None,
                  skips=None,
                  github_api_endpoint=None,
+                 pullrequest_ref=None,
                  token=None,
                  debug=False,
                  verify=False):
@@ -55,6 +56,7 @@
         self._strict = strict
         self._token = token
         self._codebase = codebase
+        self.pullrequest_ref = pullrequest_ref
         self.github_property_whitelist = github_property_whitelist
         self.skips = skips
         self.github_api_endpoint = github_api_endpoint
@@ -156,7 +158,7 @@
     def handle_pull_request(self, payload, event):
         changes = []
         number = payload['number']
-        refname = 'refs/pull/{}/merge'.format(number)
+        refname = 'refs/pull/{}/{}'.format(number, self.pullrequest_ref)
         commits = payload['pull_request']['commits']
         title = payload['pull_request']['title']
         comments = payload['pull_request']['body']
@@ -327,6 +329,7 @@
             'github_property_whitelist': options.get('github_property_whitelist', None),
             'skips': options.get('skips', None),
             'github_api_endpoint': options.get('github_api_endpoint', None) or 'https://api.github.com',
+            'pullrequest_ref': options.get('pullrequest_ref', None) or 'merge',
             'token': options.get('token', None),
             'debug': options.get('debug', None) or False,
             'verify': options.get('verify', None) or False,

{"golden_diff": "diff --git a/master/buildbot/www/hooks/github.py b/master/buildbot/www/hooks/github.py\n--- a/master/buildbot/www/hooks/github.py\n+++ b/master/buildbot/www/hooks/github.py\n@@ -48,6 +48,7 @@\n                  master=None,\n                  skips=None,\n                  github_api_endpoint=None,\n+                 pullrequest_ref=None,\n                  token=None,\n                  debug=False,\n                  verify=False):\n@@ -55,6 +56,7 @@\n         self._strict = strict\n         self._token = token\n         self._codebase = codebase\n+        self.pullrequest_ref = pullrequest_ref\n         self.github_property_whitelist = github_property_whitelist\n         self.skips = skips\n         self.github_api_endpoint = github_api_endpoint\n@@ -156,7 +158,7 @@\n     def handle_pull_request(self, payload, event):\n         changes = []\n         number = payload['number']\n-        refname = 'refs/pull/{}/merge'.format(number)\n+        refname = 'refs/pull/{}/{}'.format(number, self.pullrequest_ref)\n         commits = payload['pull_request']['commits']\n         title = payload['pull_request']['title']\n         comments = payload['pull_request']['body']\n@@ -327,6 +329,7 @@\n             'github_property_whitelist': options.get('github_property_whitelist', None),\n             'skips': options.get('skips', None),\n             'github_api_endpoint': options.get('github_api_endpoint', None) or 'https://api.github.com',\n+            'pullrequest_ref': options.get('pullrequest_ref', None) or 'merge',\n             'token': options.get('token', None),\n             'debug': options.get('debug', None) or False,\n             'verify': options.get('verify', None) or False,\n", "issue": "GitHub source step fails on missing magic `/merge` ref of GitHub\nCurrently I run into an issue where the changehook is too fast for the magic `merge` ref in GitHub to be created on Pull Requests.\r\n\r\nAn easy fix would be to try again in this case or wait a reasonable amount in `startVC()` of the GitHub step if the remote ref ends in `merge`.\r\n\r\nThoughts?\r\n\n", "before_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\n\nimport hmac\nimport json\nimport logging\nimport re\nfrom hashlib import sha1\n\nfrom dateutil.parser import parse as dateparse\n\nfrom twisted.internet import defer\nfrom twisted.python import log\n\nfrom buildbot.changes.github import PullRequestMixin\nfrom buildbot.util import bytes2unicode\nfrom buildbot.util import httpclientservice\nfrom buildbot.util import unicode2bytes\nfrom buildbot.www.hooks.base import BaseHookHandler\n\n_HEADER_EVENT = b'X-GitHub-Event'\n_HEADER_SIGNATURE = b'X-Hub-Signature'\n\nDEFAULT_SKIPS_PATTERN = (r'\\[ *skip *ci *\\]', r'\\[ *ci *skip *\\]')\nDEFAULT_GITHUB_API_URL = 'https://api.github.com'\n\n\nclass GitHubEventHandler(PullRequestMixin):\n\n    def __init__(self, secret, strict,\n                 codebase=None,\n                 github_property_whitelist=None,\n                 master=None,\n                 skips=None,\n                 github_api_endpoint=None,\n                 token=None,\n                 debug=False,\n                 verify=False):\n        self._secret = secret\n        self._strict = strict\n        self._token = token\n        self._codebase = codebase\n        self.github_property_whitelist = github_property_whitelist\n        self.skips = skips\n        self.github_api_endpoint = github_api_endpoint\n        self.master = master\n        if github_property_whitelist is None:\n            self.github_property_whitelist = []\n        if skips is None:\n            self.skips = DEFAULT_SKIPS_PATTERN\n        if github_api_endpoint is None:\n            self.github_api_endpoint = DEFAULT_GITHUB_API_URL\n\n        if self._strict and not self._secret:\n            raise ValueError('Strict mode is requested '\n                             'while no secret is provided')\n        self.debug = debug\n        self.verify = verify\n\n    @defer.inlineCallbacks\n    def process(self, request):\n        payload = self._get_payload(request)\n\n        event_type = request.getHeader(_HEADER_EVENT)\n        event_type = bytes2unicode(event_type)\n        log.msg(\"X-GitHub-Event: {}\".format(\n            event_type), logLevel=logging.DEBUG)\n\n        handler = getattr(self, 'handle_{}'.format(event_type), None)\n\n        if handler is None:\n            raise ValueError('Unknown event: {}'.format(event_type))\n\n        result = yield defer.maybeDeferred(lambda: handler(payload, event_type))\n        defer.returnValue(result)\n\n    def _get_payload(self, request):\n        content = request.content.read()\n        content = bytes2unicode(content)\n\n        signature = request.getHeader(_HEADER_SIGNATURE)\n        signature = bytes2unicode(signature)\n\n        if not signature and self._strict:\n            raise ValueError('Request has no required signature')\n\n        if self._secret and signature:\n            try:\n                hash_type, hexdigest = signature.split('=')\n            except ValueError:\n                raise ValueError(\n                    'Wrong signature format: {}'.format(signature))\n\n            if hash_type != 'sha1':\n                raise ValueError('Unknown hash type: {}'.format(hash_type))\n\n            mac = hmac.new(unicode2bytes(self._secret),\n                           msg=unicode2bytes(content),\n                           digestmod=sha1)\n            # NOTE: hmac.compare_digest should be used, but it's only available\n            # starting Python 2.7.7\n            if mac.hexdigest() != hexdigest:\n                raise ValueError('Hash mismatch')\n\n        content_type = request.getHeader(b'Content-Type')\n\n        if content_type == b'application/json':\n            payload = json.loads(content)\n        elif content_type == b'application/x-www-form-urlencoded':\n            payload = json.loads(bytes2unicode(request.args[b'payload'][0]))\n        else:\n            raise ValueError('Unknown content type: {}'.format(content_type))\n\n        log.msg(\"Payload: {}\".format(payload), logLevel=logging.DEBUG)\n\n        return payload\n\n    def handle_ping(self, _, __):\n        return [], 'git'\n\n    def handle_push(self, payload, event):\n        # This field is unused:\n        user = None\n        # user = payload['pusher']['name']\n        repo = payload['repository']['name']\n        repo_url = payload['repository']['html_url']\n        # NOTE: what would be a reasonable value for project?\n        # project = request.args.get('project', [''])[0]\n        project = payload['repository']['full_name']\n\n        # Inject some additional white-listed event payload properties\n        properties = self.extractProperties(payload)\n        changes = self._process_change(payload, user, repo, repo_url, project,\n                                       event, properties)\n\n        log.msg(\"Received {} changes from github\".format(len(changes)))\n\n        return changes, 'git'\n\n    @defer.inlineCallbacks\n    def handle_pull_request(self, payload, event):\n        changes = []\n        number = payload['number']\n        refname = 'refs/pull/{}/merge'.format(number)\n        commits = payload['pull_request']['commits']\n        title = payload['pull_request']['title']\n        comments = payload['pull_request']['body']\n        repo_full_name = payload['repository']['full_name']\n        head_sha = payload['pull_request']['head']['sha']\n\n        log.msg('Processing GitHub PR #{}'.format(number),\n                logLevel=logging.DEBUG)\n\n        head_msg = yield self._get_commit_msg(repo_full_name, head_sha)\n        if self._has_skip(head_msg):\n            log.msg(\"GitHub PR #{}, Ignoring: \"\n                    \"head commit message contains skip pattern\".format(number))\n            defer.returnValue(([], 'git'))\n\n        action = payload.get('action')\n        if action not in ('opened', 'reopened', 'synchronize'):\n            log.msg(\"GitHub PR #{} {}, ignoring\".format(number, action))\n            defer.returnValue((changes, 'git'))\n\n        properties = self.extractProperties(payload['pull_request'])\n        properties.update({'event': event})\n        change = {\n            'revision': payload['pull_request']['head']['sha'],\n            'when_timestamp': dateparse(payload['pull_request']['created_at']),\n            'branch': refname,\n            'revlink': payload['pull_request']['_links']['html']['href'],\n            'repository': payload['repository']['html_url'],\n            'project': payload['pull_request']['base']['repo']['full_name'],\n            'category': 'pull',\n            # TODO: Get author name based on login id using txgithub module\n            'author': payload['sender']['login'],\n            'comments': u'GitHub Pull Request #{0} ({1} commit{2})\\n{3}\\n{4}'.format(\n                number, commits, 's' if commits != 1 else '', title, comments),\n            'properties': properties,\n        }\n\n        if callable(self._codebase):\n            change['codebase'] = self._codebase(payload)\n        elif self._codebase is not None:\n            change['codebase'] = self._codebase\n\n        changes.append(change)\n\n        log.msg(\"Received {} changes from GitHub PR #{}\".format(\n            len(changes), number))\n        defer.returnValue((changes, 'git'))\n\n    @defer.inlineCallbacks\n    def _get_commit_msg(self, repo, sha):\n        '''\n        :param repo: the repo full name, ``{owner}/{project}``.\n            e.g. ``buildbot/buildbot``\n        '''\n        headers = {\n            'User-Agent': 'Buildbot'\n        }\n        if self._token:\n            headers['Authorization'] = 'token ' + self._token\n\n        url = '/repos/{}/commits/{}'.format(repo, sha)\n        http = yield httpclientservice.HTTPClientService.getService(\n            self.master, self.github_api_endpoint, headers=headers,\n            debug=self.debug, verify=self.verify)\n        res = yield http.get(url)\n        data = yield res.json()\n        msg = data['commit']['message']\n        defer.returnValue(msg)\n\n    def _process_change(self, payload, user, repo, repo_url, project, event,\n                        properties):\n        \"\"\"\n        Consumes the JSON as a python object and actually starts the build.\n\n        :arguments:\n            payload\n                Python Object that represents the JSON sent by GitHub Service\n                Hook.\n        \"\"\"\n        changes = []\n        refname = payload['ref']\n\n        # We only care about regular heads or tags\n        match = re.match(r\"^refs/(heads|tags)/(.+)$\", refname)\n        if not match:\n            log.msg(\"Ignoring refname `{}': Not a branch\".format(refname))\n            return changes\n        category = None  # None is the legacy category for when hook only supported push\n        if match.group(1) == \"tags\":\n            category = \"tag\"\n\n        branch = match.group(2)\n        if payload.get('deleted'):\n            log.msg(\"Branch `{}' deleted, ignoring\".format(branch))\n            return changes\n\n        # check skip pattern in commit message. e.g.: [ci skip] and [skip ci]\n        head_msg = payload['head_commit'].get('message', '')\n        if self._has_skip(head_msg):\n            return changes\n        commits = payload['commits']\n        if payload.get('created'):\n            commits = [payload['head_commit']]\n        for commit in commits:\n            files = []\n            for kind in ('added', 'modified', 'removed'):\n                files.extend(commit.get(kind, []))\n\n            when_timestamp = dateparse(commit['timestamp'])\n\n            log.msg(\"New revision: {}\".format(commit['id'][:8]))\n\n            change = {\n                'author': u'{} <{}>'.format(commit['author']['name'],\n                                           commit['author']['email']),\n                'files': files,\n                'comments': commit['message'],\n                'revision': commit['id'],\n                'when_timestamp': when_timestamp,\n                'branch': branch,\n                'revlink': commit['url'],\n                'repository': repo_url,\n                'project': project,\n                'properties': {\n                    'github_distinct': commit.get('distinct', True),\n                    'event': event,\n                },\n                'category': category\n            }\n            # Update with any white-listed github event properties\n            change['properties'].update(properties)\n\n            if callable(self._codebase):\n                change['codebase'] = self._codebase(payload)\n            elif self._codebase is not None:\n                change['codebase'] = self._codebase\n\n            changes.append(change)\n\n        return changes\n\n    def _has_skip(self, msg):\n        '''\n        The message contains the skipping keyword no not.\n\n        :return type: Bool\n        '''\n        for skip in self.skips:\n            if re.search(skip, msg):\n                return True\n        return False\n\n# for GitHub, we do another level of indirection because\n# we already had documented API that encouraged people to subclass GitHubEventHandler\n# so we need to be careful not breaking that API.\n\n\nclass GitHubHandler(BaseHookHandler):\n    def __init__(self, master, options):\n        if options is None:\n            options = {}\n        BaseHookHandler.__init__(self, master, options)\n\n        klass = options.get('class', GitHubEventHandler)\n        klass_kwargs = {\n            'master': master,\n            'codebase': options.get('codebase', None),\n            'github_property_whitelist': options.get('github_property_whitelist', None),\n            'skips': options.get('skips', None),\n            'github_api_endpoint': options.get('github_api_endpoint', None) or 'https://api.github.com',\n            'token': options.get('token', None),\n            'debug': options.get('debug', None) or False,\n            'verify': options.get('verify', None) or False,\n        }\n        handler = klass(options.get('secret', None),\n                        options.get('strict', False),\n                        **klass_kwargs)\n        self.handler = handler\n\n    def getChanges(self, request):\n        return self.handler.process(request)\n\n\ngithub = GitHubHandler\n", "path": "master/buildbot/www/hooks/github.py"}], "after_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\n\nimport hmac\nimport json\nimport logging\nimport re\nfrom hashlib import sha1\n\nfrom dateutil.parser import parse as dateparse\n\nfrom twisted.internet import defer\nfrom twisted.python import log\n\nfrom buildbot.changes.github import PullRequestMixin\nfrom buildbot.util import bytes2unicode\nfrom buildbot.util import httpclientservice\nfrom buildbot.util import unicode2bytes\nfrom buildbot.www.hooks.base import BaseHookHandler\n\n_HEADER_EVENT = b'X-GitHub-Event'\n_HEADER_SIGNATURE = b'X-Hub-Signature'\n\nDEFAULT_SKIPS_PATTERN = (r'\\[ *skip *ci *\\]', r'\\[ *ci *skip *\\]')\nDEFAULT_GITHUB_API_URL = 'https://api.github.com'\n\n\nclass GitHubEventHandler(PullRequestMixin):\n\n    def __init__(self, secret, strict,\n                 codebase=None,\n                 github_property_whitelist=None,\n                 master=None,\n                 skips=None,\n                 github_api_endpoint=None,\n                 pullrequest_ref=None,\n                 token=None,\n                 debug=False,\n                 verify=False):\n        self._secret = secret\n        self._strict = strict\n        self._token = token\n        self._codebase = codebase\n        self.pullrequest_ref = pullrequest_ref\n        self.github_property_whitelist = github_property_whitelist\n        self.skips = skips\n        self.github_api_endpoint = github_api_endpoint\n        self.master = master\n        if github_property_whitelist is None:\n            self.github_property_whitelist = []\n        if skips is None:\n            self.skips = DEFAULT_SKIPS_PATTERN\n        if github_api_endpoint is None:\n            self.github_api_endpoint = DEFAULT_GITHUB_API_URL\n\n        if self._strict and not self._secret:\n            raise ValueError('Strict mode is requested '\n                             'while no secret is provided')\n        self.debug = debug\n        self.verify = verify\n\n    @defer.inlineCallbacks\n    def process(self, request):\n        payload = self._get_payload(request)\n\n        event_type = request.getHeader(_HEADER_EVENT)\n        event_type = bytes2unicode(event_type)\n        log.msg(\"X-GitHub-Event: {}\".format(\n            event_type), logLevel=logging.DEBUG)\n\n        handler = getattr(self, 'handle_{}'.format(event_type), None)\n\n        if handler is None:\n            raise ValueError('Unknown event: {}'.format(event_type))\n\n        result = yield defer.maybeDeferred(lambda: handler(payload, event_type))\n        defer.returnValue(result)\n\n    def _get_payload(self, request):\n        content = request.content.read()\n        content = bytes2unicode(content)\n\n        signature = request.getHeader(_HEADER_SIGNATURE)\n        signature = bytes2unicode(signature)\n\n        if not signature and self._strict:\n            raise ValueError('Request has no required signature')\n\n        if self._secret and signature:\n            try:\n                hash_type, hexdigest = signature.split('=')\n            except ValueError:\n                raise ValueError(\n                    'Wrong signature format: {}'.format(signature))\n\n            if hash_type != 'sha1':\n                raise ValueError('Unknown hash type: {}'.format(hash_type))\n\n            mac = hmac.new(unicode2bytes(self._secret),\n                           msg=unicode2bytes(content),\n                           digestmod=sha1)\n            # NOTE: hmac.compare_digest should be used, but it's only available\n            # starting Python 2.7.7\n            if mac.hexdigest() != hexdigest:\n                raise ValueError('Hash mismatch')\n\n        content_type = request.getHeader(b'Content-Type')\n\n        if content_type == b'application/json':\n            payload = json.loads(content)\n        elif content_type == b'application/x-www-form-urlencoded':\n            payload = json.loads(bytes2unicode(request.args[b'payload'][0]))\n        else:\n            raise ValueError('Unknown content type: {}'.format(content_type))\n\n        log.msg(\"Payload: {}\".format(payload), logLevel=logging.DEBUG)\n\n        return payload\n\n    def handle_ping(self, _, __):\n        return [], 'git'\n\n    def handle_push(self, payload, event):\n        # This field is unused:\n        user = None\n        # user = payload['pusher']['name']\n        repo = payload['repository']['name']\n        repo_url = payload['repository']['html_url']\n        # NOTE: what would be a reasonable value for project?\n        # project = request.args.get('project', [''])[0]\n        project = payload['repository']['full_name']\n\n        # Inject some additional white-listed event payload properties\n        properties = self.extractProperties(payload)\n        changes = self._process_change(payload, user, repo, repo_url, project,\n                                       event, properties)\n\n        log.msg(\"Received {} changes from github\".format(len(changes)))\n\n        return changes, 'git'\n\n    @defer.inlineCallbacks\n    def handle_pull_request(self, payload, event):\n        changes = []\n        number = payload['number']\n        refname = 'refs/pull/{}/{}'.format(number, self.pullrequest_ref)\n        commits = payload['pull_request']['commits']\n        title = payload['pull_request']['title']\n        comments = payload['pull_request']['body']\n        repo_full_name = payload['repository']['full_name']\n        head_sha = payload['pull_request']['head']['sha']\n\n        log.msg('Processing GitHub PR #{}'.format(number),\n                logLevel=logging.DEBUG)\n\n        head_msg = yield self._get_commit_msg(repo_full_name, head_sha)\n        if self._has_skip(head_msg):\n            log.msg(\"GitHub PR #{}, Ignoring: \"\n                    \"head commit message contains skip pattern\".format(number))\n            defer.returnValue(([], 'git'))\n\n        action = payload.get('action')\n        if action not in ('opened', 'reopened', 'synchronize'):\n            log.msg(\"GitHub PR #{} {}, ignoring\".format(number, action))\n            defer.returnValue((changes, 'git'))\n\n        properties = self.extractProperties(payload['pull_request'])\n        properties.update({'event': event})\n        change = {\n            'revision': payload['pull_request']['head']['sha'],\n            'when_timestamp': dateparse(payload['pull_request']['created_at']),\n            'branch': refname,\n            'revlink': payload['pull_request']['_links']['html']['href'],\n            'repository': payload['repository']['html_url'],\n            'project': payload['pull_request']['base']['repo']['full_name'],\n            'category': 'pull',\n            # TODO: Get author name based on login id using txgithub module\n            'author': payload['sender']['login'],\n            'comments': u'GitHub Pull Request #{0} ({1} commit{2})\\n{3}\\n{4}'.format(\n                number, commits, 's' if commits != 1 else '', title, comments),\n            'properties': properties,\n        }\n\n        if callable(self._codebase):\n            change['codebase'] = self._codebase(payload)\n        elif self._codebase is not None:\n            change['codebase'] = self._codebase\n\n        changes.append(change)\n\n        log.msg(\"Received {} changes from GitHub PR #{}\".format(\n            len(changes), number))\n        defer.returnValue((changes, 'git'))\n\n    @defer.inlineCallbacks\n    def _get_commit_msg(self, repo, sha):\n        '''\n        :param repo: the repo full name, ``{owner}/{project}``.\n            e.g. ``buildbot/buildbot``\n        '''\n        headers = {\n            'User-Agent': 'Buildbot'\n        }\n        if self._token:\n            headers['Authorization'] = 'token ' + self._token\n\n        url = '/repos/{}/commits/{}'.format(repo, sha)\n        http = yield httpclientservice.HTTPClientService.getService(\n            self.master, self.github_api_endpoint, headers=headers,\n            debug=self.debug, verify=self.verify)\n        res = yield http.get(url)\n        data = yield res.json()\n        msg = data['commit']['message']\n        defer.returnValue(msg)\n\n    def _process_change(self, payload, user, repo, repo_url, project, event,\n                        properties):\n        \"\"\"\n        Consumes the JSON as a python object and actually starts the build.\n\n        :arguments:\n            payload\n                Python Object that represents the JSON sent by GitHub Service\n                Hook.\n        \"\"\"\n        changes = []\n        refname = payload['ref']\n\n        # We only care about regular heads or tags\n        match = re.match(r\"^refs/(heads|tags)/(.+)$\", refname)\n        if not match:\n            log.msg(\"Ignoring refname `{}': Not a branch\".format(refname))\n            return changes\n        category = None  # None is the legacy category for when hook only supported push\n        if match.group(1) == \"tags\":\n            category = \"tag\"\n\n        branch = match.group(2)\n        if payload.get('deleted'):\n            log.msg(\"Branch `{}' deleted, ignoring\".format(branch))\n            return changes\n\n        # check skip pattern in commit message. e.g.: [ci skip] and [skip ci]\n        head_msg = payload['head_commit'].get('message', '')\n        if self._has_skip(head_msg):\n            return changes\n        commits = payload['commits']\n        if payload.get('created'):\n            commits = [payload['head_commit']]\n        for commit in commits:\n            files = []\n            for kind in ('added', 'modified', 'removed'):\n                files.extend(commit.get(kind, []))\n\n            when_timestamp = dateparse(commit['timestamp'])\n\n            log.msg(\"New revision: {}\".format(commit['id'][:8]))\n\n            change = {\n                'author': u'{} <{}>'.format(commit['author']['name'],\n                                           commit['author']['email']),\n                'files': files,\n                'comments': commit['message'],\n                'revision': commit['id'],\n                'when_timestamp': when_timestamp,\n                'branch': branch,\n                'revlink': commit['url'],\n                'repository': repo_url,\n                'project': project,\n                'properties': {\n                    'github_distinct': commit.get('distinct', True),\n                    'event': event,\n                },\n                'category': category\n            }\n            # Update with any white-listed github event properties\n            change['properties'].update(properties)\n\n            if callable(self._codebase):\n                change['codebase'] = self._codebase(payload)\n            elif self._codebase is not None:\n                change['codebase'] = self._codebase\n\n            changes.append(change)\n\n        return changes\n\n    def _has_skip(self, msg):\n        '''\n        The message contains the skipping keyword no not.\n\n        :return type: Bool\n        '''\n        for skip in self.skips:\n            if re.search(skip, msg):\n                return True\n        return False\n\n# for GitHub, we do another level of indirection because\n# we already had documented API that encouraged people to subclass GitHubEventHandler\n# so we need to be careful not breaking that API.\n\n\nclass GitHubHandler(BaseHookHandler):\n    def __init__(self, master, options):\n        if options is None:\n            options = {}\n        BaseHookHandler.__init__(self, master, options)\n\n        klass = options.get('class', GitHubEventHandler)\n        klass_kwargs = {\n            'master': master,\n            'codebase': options.get('codebase', None),\n            'github_property_whitelist': options.get('github_property_whitelist', None),\n            'skips': options.get('skips', None),\n            'github_api_endpoint': options.get('github_api_endpoint', None) or 'https://api.github.com',\n            'pullrequest_ref': options.get('pullrequest_ref', None) or 'merge',\n            'token': options.get('token', None),\n            'debug': options.get('debug', None) or False,\n            'verify': options.get('verify', None) or False,\n        }\n        handler = klass(options.get('secret', None),\n                        options.get('strict', False),\n                        **klass_kwargs)\n        self.handler = handler\n\n    def getChanges(self, request):\n        return self.handler.process(request)\n\n\ngithub = GitHubHandler\n", "path": "master/buildbot/www/hooks/github.py"}]}