We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
verify that all entry points of Buildbot are at least importable
This ticket is a migrated Trac ticket [3470](http://trac.buildbot.net/ticket/3470)

People contributed to the original ticket: @rutsky
Ticket created on: `Feb 24 2016`
Ticket last modified on: `Feb 24 2016`

---

We have big bunch of plugins exported in setup.py, and looks like not all plugins are exported correctly: https://github.com/buildbot/buildbot/pull/2006/files#diff-9030ee1b7d992477393062826c5decfeL345

It would be nice to have test or check for that.


---




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `master/buildbot/db/schedulers.py`
Content:
```
1 # This file is part of Buildbot.  Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 import sqlalchemy as sa
17 import sqlalchemy.exc
18 
19 from buildbot.db import NULL
20 from buildbot.db import base
21 from twisted.internet import defer
22 
23 
24 class SchedulerAlreadyClaimedError(Exception):
25     pass
26 
27 
28 class SchedulersConnectorComponent(base.DBConnectorComponent):
29     # Documentation is in developer/db.rst
30 
31     def classifyChanges(self, schedulerid, classifications):
32         def thd(conn):
33             transaction = conn.begin()
34             tbl = self.db.model.scheduler_changes
35             ins_q = tbl.insert()
36             upd_q = tbl.update(
37                 ((tbl.c.schedulerid == schedulerid)
38                  & (tbl.c.changeid == sa.bindparam('wc_changeid'))))
39             for changeid, important in classifications.items():
40                 # convert the 'important' value into an integer, since that
41                 # is the column type
42                 imp_int = important and 1 or 0
43                 try:
44                     conn.execute(ins_q,
45                                  schedulerid=schedulerid,
46                                  changeid=changeid,
47                                  important=imp_int)
48                 except (sqlalchemy.exc.ProgrammingError,
49                         sqlalchemy.exc.IntegrityError):
50                     # insert failed, so try an update
51                     conn.execute(upd_q,
52                                  wc_changeid=changeid,
53                                  important=imp_int)
54 
55             transaction.commit()
56         return self.db.pool.do(thd)
57 
58     def flushChangeClassifications(self, schedulerid, less_than=None):
59         def thd(conn):
60             sch_ch_tbl = self.db.model.scheduler_changes
61             wc = (sch_ch_tbl.c.schedulerid == schedulerid)
62             if less_than is not None:
63                 wc = wc & (sch_ch_tbl.c.changeid < less_than)
64             q = sch_ch_tbl.delete(whereclause=wc)
65             conn.execute(q)
66         return self.db.pool.do(thd)
67 
68     def getChangeClassifications(self, schedulerid, branch=-1,
69                                  repository=-1, project=-1,
70                                  codebase=-1):
71         # -1 here stands for "argument not given", since None has meaning
72         # as a branch
73         def thd(conn):
74             sch_ch_tbl = self.db.model.scheduler_changes
75             ch_tbl = self.db.model.changes
76 
77             wc = (sch_ch_tbl.c.schedulerid == schedulerid)
78 
79             # may need to filter further based on branch, etc
80             extra_wheres = []
81             if branch != -1:
82                 extra_wheres.append(ch_tbl.c.branch == branch)
83             if repository != -1:
84                 extra_wheres.append(ch_tbl.c.repository == repository)
85             if project != -1:
86                 extra_wheres.append(ch_tbl.c.project == project)
87             if codebase != -1:
88                 extra_wheres.append(ch_tbl.c.codebase == codebase)
89 
90             # if we need to filter further append those, as well as a join
91             # on changeid (but just once for that one)
92             if extra_wheres:
93                 wc &= (sch_ch_tbl.c.changeid == ch_tbl.c.changeid)
94                 for w in extra_wheres:
95                     wc &= w
96 
97             q = sa.select(
98                 [sch_ch_tbl.c.changeid, sch_ch_tbl.c.important],
99                 whereclause=wc)
100             return dict([(r.changeid, [False, True][r.important])
101                          for r in conn.execute(q)])
102         return self.db.pool.do(thd)
103 
104     def findSchedulerId(self, name):
105         tbl = self.db.model.schedulers
106         name_hash = self.hashColumns(name)
107         return self.findSomethingId(
108             tbl=tbl,
109             whereclause=(tbl.c.name_hash == name_hash),
110             insert_values=dict(
111                 name=name,
112                 name_hash=name_hash,
113             ))
114 
115     def setSchedulerMaster(self, schedulerid, masterid):
116         def thd(conn):
117             sch_mst_tbl = self.db.model.scheduler_masters
118 
119             # handle the masterid=None case to get it out of the way
120             if masterid is None:
121                 q = sch_mst_tbl.delete(
122                     whereclause=(sch_mst_tbl.c.schedulerid == schedulerid))
123                 conn.execute(q)
124                 return
125 
126             # try a blind insert..
127             try:
128                 q = sch_mst_tbl.insert()
129                 conn.execute(q,
130                              dict(schedulerid=schedulerid, masterid=masterid))
131             except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
132                 # someone already owns this scheduler.
133                 raise SchedulerAlreadyClaimedError
134 
135         return self.db.pool.do(thd)
136 
137     @defer.inlineCallbacks
138     def getScheduler(self, schedulerid):
139         sch = yield self.getSchedulers(_schedulerid=schedulerid)
140         if sch:
141             defer.returnValue(sch[0])
142 
143     def getSchedulers(self, active=None, masterid=None, _schedulerid=None):
144         def thd(conn):
145             sch_tbl = self.db.model.schedulers
146             sch_mst_tbl = self.db.model.scheduler_masters
147 
148             # handle the trivial case of masterid=xx and active=False
149             if masterid is not None and active is not None and not active:
150                 return []
151 
152             join = sch_tbl.outerjoin(sch_mst_tbl,
153                                      (sch_tbl.c.id == sch_mst_tbl.c.schedulerid))
154 
155             # if we're given a _schedulerid, select only that row
156             wc = None
157             if _schedulerid:
158                 wc = (sch_tbl.c.id == _schedulerid)
159             else:
160                 # otherwise, filter with active, if necessary
161                 if masterid is not None:
162                     wc = (sch_mst_tbl.c.masterid == masterid)
163                 elif active:
164                     wc = (sch_mst_tbl.c.masterid != NULL)
165                 elif active is not None:
166                     wc = (sch_mst_tbl.c.masterid == NULL)
167 
168             q = sa.select([sch_tbl.c.id, sch_tbl.c.name,
169                            sch_mst_tbl.c.masterid],
170                           from_obj=join, whereclause=wc)
171 
172             return [dict(id=row.id, name=row.name,
173                          masterid=row.masterid)
174                     for row in conn.execute(q).fetchall()]
175         return self.db.pool.do(thd)
176
```

Path: `master/buildbot/db/migrate/versions/037_buildrequests_builderid.py`
Content:
```
1 # This file is part of Buildbot.  Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 import hashlib
17 import sqlalchemy as sa
18 
19 from migrate import changeset
20 
21 
22 def add_new_schema_parts(migrate_engine):
23     metadata = sa.MetaData()
24     metadata.bind = migrate_engine
25     sa.Table('builders', metadata, autoload=True)
26     sa.Table('buildsets', metadata, autoload=True)
27 
28     buildrequests = sa.Table('buildrequests', metadata, autoload=True)
29 
30     builderid = sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'))
31     builderid.create(buildrequests)
32 
33     # Remove all index
34     for index in buildrequests.indexes:
35         index.drop()
36 
37 
38 def migrate_data(migrate_engine):
39     metadata = sa.MetaData()
40     metadata.bind = migrate_engine
41 
42     # set up the tables we'll need to migrate
43     buildrequests = sa.Table('buildrequests', metadata, autoload=True)
44     builders = sa.Table('builders', metadata, autoload=True)
45 
46     bName2bID = dict()
47     q = sa.select([builders.c.id, builders.c.name])
48     for row in migrate_engine.execute(q).fetchall():
49         bName2bID[row.name] = row.id
50 
51     def hashColumns(*args):
52         # copy paste from buildbot/db/base.py
53         def encode(x):
54             try:
55                 return x.encode('utf8')
56             except AttributeError:
57                 if x is None:
58                     return '\xf5'
59                 return str(x)
60         return hashlib.sha1('\0'.join(map(encode, args))).hexdigest()
61 
62     def findbuilderid(buildername):
63         bid = bName2bID.get(buildername)
64         if bid is None:
65             r = migrate_engine.execute(builders.insert(), [{
66                 'name': buildername,
67                 'name_hash': hashColumns(buildername),
68             }])
69             bid = r.inserted_primary_key[0]
70             bName2bID[buildername] = bid
71         return bid
72 
73     c = buildrequests.c
74     q = sa.select([c.id, c.buildername])
75     for row in migrate_engine.execute(q).fetchall():
76         builderid = findbuilderid(row.buildername)
77         migrate_engine.execute(
78             buildrequests.update(whereclause=(c.id == row.id)),
79             builderid=builderid)
80 
81 
82 def remove_buildername(migrate_engine):
83 
84     metadata = sa.MetaData()
85     metadata.bind = migrate_engine
86 
87     sa.Table('builders', metadata, autoload=True)
88     sa.Table('buildsets', metadata, autoload=True)
89 
90     # Specify what the new table should look like
91     buildrequests = sa.Table('buildrequests', metadata,
92                              sa.Column('id', sa.Integer, primary_key=True),
93                              sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"),
94                                        nullable=False),
95                              sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'),
96                                        nullable=False),
97                              sa.Column('priority', sa.Integer, nullable=False,
98                                        server_default=sa.DefaultClause("0")),
99                              sa.Column('complete', sa.Integer,
100                                        server_default=sa.DefaultClause("0")),
101                              sa.Column('results', sa.SmallInteger),
102                              sa.Column('submitted_at', sa.Integer, nullable=False),
103                              sa.Column('complete_at', sa.Integer),
104                              sa.Column('waited_for', sa.SmallInteger,
105                                        server_default=sa.DefaultClause("0")),
106                              )
107     changeset.drop_column(
108         sa.Column('buildername', sa.String(length=256), nullable=False),
109         table=buildrequests,
110         metadata=metadata,
111         engine=migrate_engine)
112 
113     changeset.alter_column(
114         sa.Column('builderid', sa.Integer, sa.ForeignKey("builders.id"), nullable=False),
115         table='buildrequests',
116         metadata=metadata,
117         engine=migrate_engine)
118 
119     idx = sa.Index('buildrequests_builderid', buildrequests.c.builderid)
120     idx.create(migrate_engine)
121     idx = sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid)
122     idx.create(migrate_engine)
123     idx = sa.Index('buildrequests_complete', buildrequests.c.complete)
124     idx.create(migrate_engine)
125 
126 
127 def upgrade(migrate_engine):
128     # add a 'builderid' column to buildrequests
129     add_new_schema_parts(migrate_engine)
130     # migrate the data to new tables
131     migrate_data(migrate_engine)
132 
133     # Finally remove the buildername column
134     remove_buildername(migrate_engine)
135
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py b/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py
--- a/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py
+++ b/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py
@@ -86,33 +86,14 @@
 
     sa.Table('builders', metadata, autoload=True)
     sa.Table('buildsets', metadata, autoload=True)
+    buildrequests = sa.Table('buildrequests', metadata, autoload=True)
 
     # Specify what the new table should look like
-    buildrequests = sa.Table('buildrequests', metadata,
-                             sa.Column('id', sa.Integer, primary_key=True),
-                             sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"),
-                                       nullable=False),
-                             sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'),
-                                       nullable=False),
-                             sa.Column('priority', sa.Integer, nullable=False,
-                                       server_default=sa.DefaultClause("0")),
-                             sa.Column('complete', sa.Integer,
-                                       server_default=sa.DefaultClause("0")),
-                             sa.Column('results', sa.SmallInteger),
-                             sa.Column('submitted_at', sa.Integer, nullable=False),
-                             sa.Column('complete_at', sa.Integer),
-                             sa.Column('waited_for', sa.SmallInteger,
-                                       server_default=sa.DefaultClause("0")),
-                             )
-    changeset.drop_column(
-        sa.Column('buildername', sa.String(length=256), nullable=False),
-        table=buildrequests,
-        metadata=metadata,
-        engine=migrate_engine)
+    buildrequests.c.buildername.drop()
 
     changeset.alter_column(
         sa.Column('builderid', sa.Integer, sa.ForeignKey("builders.id"), nullable=False),
-        table='buildrequests',
+        table=buildrequests,
         metadata=metadata,
         engine=migrate_engine)
 
@@ -129,6 +110,5 @@
     add_new_schema_parts(migrate_engine)
     # migrate the data to new tables
     migrate_data(migrate_engine)
-
     # Finally remove the buildername column
     remove_buildername(migrate_engine)
diff --git a/master/buildbot/db/schedulers.py b/master/buildbot/db/schedulers.py
--- a/master/buildbot/db/schedulers.py
+++ b/master/buildbot/db/schedulers.py
@@ -47,6 +47,8 @@
                                  important=imp_int)
                 except (sqlalchemy.exc.ProgrammingError,
                         sqlalchemy.exc.IntegrityError):
+                    transaction.rollback()
+                    transaction = conn.begin()
                     # insert failed, so try an update
                     conn.execute(upd_q,
                                  wc_changeid=changeid,

{"golden_diff": "diff --git a/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py b/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py\n--- a/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py\n+++ b/master/buildbot/db/migrate/versions/037_buildrequests_builderid.py\n@@ -86,33 +86,14 @@\n \n     sa.Table('builders', metadata, autoload=True)\n     sa.Table('buildsets', metadata, autoload=True)\n+    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n \n     # Specify what the new table should look like\n-    buildrequests = sa.Table('buildrequests', metadata,\n-                             sa.Column('id', sa.Integer, primary_key=True),\n-                             sa.Column('buildsetid', sa.Integer, sa.ForeignKey(\"buildsets.id\"),\n-                                       nullable=False),\n-                             sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'),\n-                                       nullable=False),\n-                             sa.Column('priority', sa.Integer, nullable=False,\n-                                       server_default=sa.DefaultClause(\"0\")),\n-                             sa.Column('complete', sa.Integer,\n-                                       server_default=sa.DefaultClause(\"0\")),\n-                             sa.Column('results', sa.SmallInteger),\n-                             sa.Column('submitted_at', sa.Integer, nullable=False),\n-                             sa.Column('complete_at', sa.Integer),\n-                             sa.Column('waited_for', sa.SmallInteger,\n-                                       server_default=sa.DefaultClause(\"0\")),\n-                             )\n-    changeset.drop_column(\n-        sa.Column('buildername', sa.String(length=256), nullable=False),\n-        table=buildrequests,\n-        metadata=metadata,\n-        engine=migrate_engine)\n+    buildrequests.c.buildername.drop()\n \n     changeset.alter_column(\n         sa.Column('builderid', sa.Integer, sa.ForeignKey(\"builders.id\"), nullable=False),\n-        table='buildrequests',\n+        table=buildrequests,\n         metadata=metadata,\n         engine=migrate_engine)\n \n@@ -129,6 +110,5 @@\n     add_new_schema_parts(migrate_engine)\n     # migrate the data to new tables\n     migrate_data(migrate_engine)\n-\n     # Finally remove the buildername column\n     remove_buildername(migrate_engine)\ndiff --git a/master/buildbot/db/schedulers.py b/master/buildbot/db/schedulers.py\n--- a/master/buildbot/db/schedulers.py\n+++ b/master/buildbot/db/schedulers.py\n@@ -47,6 +47,8 @@\n                                  important=imp_int)\n                 except (sqlalchemy.exc.ProgrammingError,\n                         sqlalchemy.exc.IntegrityError):\n+                    transaction.rollback()\n+                    transaction = conn.begin()\n                     # insert failed, so try an update\n                     conn.execute(upd_q,\n                                  wc_changeid=changeid,\n", "issue": "verify that all entry points of Buildbot are at least importable\nThis ticket is a migrated Trac ticket [3470](http://trac.buildbot.net/ticket/3470)\n\nPeople contributed to the original ticket: @rutsky\nTicket created on: `Feb 24 2016`\nTicket last modified on: `Feb 24 2016`\n\n---\n\nWe have big bunch of plugins exported in setup.py, and looks like not all plugins are exported correctly: https://github.com/buildbot/buildbot/pull/2006/files#diff-9030ee1b7d992477393062826c5decfeL345\n\nIt would be nice to have test or check for that.\n\n\n---\n\n\n\n", "before_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nimport sqlalchemy as sa\nimport sqlalchemy.exc\n\nfrom buildbot.db import NULL\nfrom buildbot.db import base\nfrom twisted.internet import defer\n\n\nclass SchedulerAlreadyClaimedError(Exception):\n    pass\n\n\nclass SchedulersConnectorComponent(base.DBConnectorComponent):\n    # Documentation is in developer/db.rst\n\n    def classifyChanges(self, schedulerid, classifications):\n        def thd(conn):\n            transaction = conn.begin()\n            tbl = self.db.model.scheduler_changes\n            ins_q = tbl.insert()\n            upd_q = tbl.update(\n                ((tbl.c.schedulerid == schedulerid)\n                 & (tbl.c.changeid == sa.bindparam('wc_changeid'))))\n            for changeid, important in classifications.items():\n                # convert the 'important' value into an integer, since that\n                # is the column type\n                imp_int = important and 1 or 0\n                try:\n                    conn.execute(ins_q,\n                                 schedulerid=schedulerid,\n                                 changeid=changeid,\n                                 important=imp_int)\n                except (sqlalchemy.exc.ProgrammingError,\n                        sqlalchemy.exc.IntegrityError):\n                    # insert failed, so try an update\n                    conn.execute(upd_q,\n                                 wc_changeid=changeid,\n                                 important=imp_int)\n\n            transaction.commit()\n        return self.db.pool.do(thd)\n\n    def flushChangeClassifications(self, schedulerid, less_than=None):\n        def thd(conn):\n            sch_ch_tbl = self.db.model.scheduler_changes\n            wc = (sch_ch_tbl.c.schedulerid == schedulerid)\n            if less_than is not None:\n                wc = wc & (sch_ch_tbl.c.changeid < less_than)\n            q = sch_ch_tbl.delete(whereclause=wc)\n            conn.execute(q)\n        return self.db.pool.do(thd)\n\n    def getChangeClassifications(self, schedulerid, branch=-1,\n                                 repository=-1, project=-1,\n                                 codebase=-1):\n        # -1 here stands for \"argument not given\", since None has meaning\n        # as a branch\n        def thd(conn):\n            sch_ch_tbl = self.db.model.scheduler_changes\n            ch_tbl = self.db.model.changes\n\n            wc = (sch_ch_tbl.c.schedulerid == schedulerid)\n\n            # may need to filter further based on branch, etc\n            extra_wheres = []\n            if branch != -1:\n                extra_wheres.append(ch_tbl.c.branch == branch)\n            if repository != -1:\n                extra_wheres.append(ch_tbl.c.repository == repository)\n            if project != -1:\n                extra_wheres.append(ch_tbl.c.project == project)\n            if codebase != -1:\n                extra_wheres.append(ch_tbl.c.codebase == codebase)\n\n            # if we need to filter further append those, as well as a join\n            # on changeid (but just once for that one)\n            if extra_wheres:\n                wc &= (sch_ch_tbl.c.changeid == ch_tbl.c.changeid)\n                for w in extra_wheres:\n                    wc &= w\n\n            q = sa.select(\n                [sch_ch_tbl.c.changeid, sch_ch_tbl.c.important],\n                whereclause=wc)\n            return dict([(r.changeid, [False, True][r.important])\n                         for r in conn.execute(q)])\n        return self.db.pool.do(thd)\n\n    def findSchedulerId(self, name):\n        tbl = self.db.model.schedulers\n        name_hash = self.hashColumns(name)\n        return self.findSomethingId(\n            tbl=tbl,\n            whereclause=(tbl.c.name_hash == name_hash),\n            insert_values=dict(\n                name=name,\n                name_hash=name_hash,\n            ))\n\n    def setSchedulerMaster(self, schedulerid, masterid):\n        def thd(conn):\n            sch_mst_tbl = self.db.model.scheduler_masters\n\n            # handle the masterid=None case to get it out of the way\n            if masterid is None:\n                q = sch_mst_tbl.delete(\n                    whereclause=(sch_mst_tbl.c.schedulerid == schedulerid))\n                conn.execute(q)\n                return\n\n            # try a blind insert..\n            try:\n                q = sch_mst_tbl.insert()\n                conn.execute(q,\n                             dict(schedulerid=schedulerid, masterid=masterid))\n            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):\n                # someone already owns this scheduler.\n                raise SchedulerAlreadyClaimedError\n\n        return self.db.pool.do(thd)\n\n    @defer.inlineCallbacks\n    def getScheduler(self, schedulerid):\n        sch = yield self.getSchedulers(_schedulerid=schedulerid)\n        if sch:\n            defer.returnValue(sch[0])\n\n    def getSchedulers(self, active=None, masterid=None, _schedulerid=None):\n        def thd(conn):\n            sch_tbl = self.db.model.schedulers\n            sch_mst_tbl = self.db.model.scheduler_masters\n\n            # handle the trivial case of masterid=xx and active=False\n            if masterid is not None and active is not None and not active:\n                return []\n\n            join = sch_tbl.outerjoin(sch_mst_tbl,\n                                     (sch_tbl.c.id == sch_mst_tbl.c.schedulerid))\n\n            # if we're given a _schedulerid, select only that row\n            wc = None\n            if _schedulerid:\n                wc = (sch_tbl.c.id == _schedulerid)\n            else:\n                # otherwise, filter with active, if necessary\n                if masterid is not None:\n                    wc = (sch_mst_tbl.c.masterid == masterid)\n                elif active:\n                    wc = (sch_mst_tbl.c.masterid != NULL)\n                elif active is not None:\n                    wc = (sch_mst_tbl.c.masterid == NULL)\n\n            q = sa.select([sch_tbl.c.id, sch_tbl.c.name,\n                           sch_mst_tbl.c.masterid],\n                          from_obj=join, whereclause=wc)\n\n            return [dict(id=row.id, name=row.name,\n                         masterid=row.masterid)\n                    for row in conn.execute(q).fetchall()]\n        return self.db.pool.do(thd)\n", "path": "master/buildbot/db/schedulers.py"}, {"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nimport hashlib\nimport sqlalchemy as sa\n\nfrom migrate import changeset\n\n\ndef add_new_schema_parts(migrate_engine):\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n    sa.Table('builders', metadata, autoload=True)\n    sa.Table('buildsets', metadata, autoload=True)\n\n    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n\n    builderid = sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'))\n    builderid.create(buildrequests)\n\n    # Remove all index\n    for index in buildrequests.indexes:\n        index.drop()\n\n\ndef migrate_data(migrate_engine):\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n\n    # set up the tables we'll need to migrate\n    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n    builders = sa.Table('builders', metadata, autoload=True)\n\n    bName2bID = dict()\n    q = sa.select([builders.c.id, builders.c.name])\n    for row in migrate_engine.execute(q).fetchall():\n        bName2bID[row.name] = row.id\n\n    def hashColumns(*args):\n        # copy paste from buildbot/db/base.py\n        def encode(x):\n            try:\n                return x.encode('utf8')\n            except AttributeError:\n                if x is None:\n                    return '\\xf5'\n                return str(x)\n        return hashlib.sha1('\\0'.join(map(encode, args))).hexdigest()\n\n    def findbuilderid(buildername):\n        bid = bName2bID.get(buildername)\n        if bid is None:\n            r = migrate_engine.execute(builders.insert(), [{\n                'name': buildername,\n                'name_hash': hashColumns(buildername),\n            }])\n            bid = r.inserted_primary_key[0]\n            bName2bID[buildername] = bid\n        return bid\n\n    c = buildrequests.c\n    q = sa.select([c.id, c.buildername])\n    for row in migrate_engine.execute(q).fetchall():\n        builderid = findbuilderid(row.buildername)\n        migrate_engine.execute(\n            buildrequests.update(whereclause=(c.id == row.id)),\n            builderid=builderid)\n\n\ndef remove_buildername(migrate_engine):\n\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n\n    sa.Table('builders', metadata, autoload=True)\n    sa.Table('buildsets', metadata, autoload=True)\n\n    # Specify what the new table should look like\n    buildrequests = sa.Table('buildrequests', metadata,\n                             sa.Column('id', sa.Integer, primary_key=True),\n                             sa.Column('buildsetid', sa.Integer, sa.ForeignKey(\"buildsets.id\"),\n                                       nullable=False),\n                             sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'),\n                                       nullable=False),\n                             sa.Column('priority', sa.Integer, nullable=False,\n                                       server_default=sa.DefaultClause(\"0\")),\n                             sa.Column('complete', sa.Integer,\n                                       server_default=sa.DefaultClause(\"0\")),\n                             sa.Column('results', sa.SmallInteger),\n                             sa.Column('submitted_at', sa.Integer, nullable=False),\n                             sa.Column('complete_at', sa.Integer),\n                             sa.Column('waited_for', sa.SmallInteger,\n                                       server_default=sa.DefaultClause(\"0\")),\n                             )\n    changeset.drop_column(\n        sa.Column('buildername', sa.String(length=256), nullable=False),\n        table=buildrequests,\n        metadata=metadata,\n        engine=migrate_engine)\n\n    changeset.alter_column(\n        sa.Column('builderid', sa.Integer, sa.ForeignKey(\"builders.id\"), nullable=False),\n        table='buildrequests',\n        metadata=metadata,\n        engine=migrate_engine)\n\n    idx = sa.Index('buildrequests_builderid', buildrequests.c.builderid)\n    idx.create(migrate_engine)\n    idx = sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid)\n    idx.create(migrate_engine)\n    idx = sa.Index('buildrequests_complete', buildrequests.c.complete)\n    idx.create(migrate_engine)\n\n\ndef upgrade(migrate_engine):\n    # add a 'builderid' column to buildrequests\n    add_new_schema_parts(migrate_engine)\n    # migrate the data to new tables\n    migrate_data(migrate_engine)\n\n    # Finally remove the buildername column\n    remove_buildername(migrate_engine)\n", "path": "master/buildbot/db/migrate/versions/037_buildrequests_builderid.py"}], "after_files": [{"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nimport sqlalchemy as sa\nimport sqlalchemy.exc\n\nfrom buildbot.db import NULL\nfrom buildbot.db import base\nfrom twisted.internet import defer\n\n\nclass SchedulerAlreadyClaimedError(Exception):\n    pass\n\n\nclass SchedulersConnectorComponent(base.DBConnectorComponent):\n    # Documentation is in developer/db.rst\n\n    def classifyChanges(self, schedulerid, classifications):\n        def thd(conn):\n            transaction = conn.begin()\n            tbl = self.db.model.scheduler_changes\n            ins_q = tbl.insert()\n            upd_q = tbl.update(\n                ((tbl.c.schedulerid == schedulerid)\n                 & (tbl.c.changeid == sa.bindparam('wc_changeid'))))\n            for changeid, important in classifications.items():\n                # convert the 'important' value into an integer, since that\n                # is the column type\n                imp_int = important and 1 or 0\n                try:\n                    conn.execute(ins_q,\n                                 schedulerid=schedulerid,\n                                 changeid=changeid,\n                                 important=imp_int)\n                except (sqlalchemy.exc.ProgrammingError,\n                        sqlalchemy.exc.IntegrityError):\n                    transaction.rollback()\n                    transaction = conn.begin()\n                    # insert failed, so try an update\n                    conn.execute(upd_q,\n                                 wc_changeid=changeid,\n                                 important=imp_int)\n\n            transaction.commit()\n        return self.db.pool.do(thd)\n\n    def flushChangeClassifications(self, schedulerid, less_than=None):\n        def thd(conn):\n            sch_ch_tbl = self.db.model.scheduler_changes\n            wc = (sch_ch_tbl.c.schedulerid == schedulerid)\n            if less_than is not None:\n                wc = wc & (sch_ch_tbl.c.changeid < less_than)\n            q = sch_ch_tbl.delete(whereclause=wc)\n            conn.execute(q)\n        return self.db.pool.do(thd)\n\n    def getChangeClassifications(self, schedulerid, branch=-1,\n                                 repository=-1, project=-1,\n                                 codebase=-1):\n        # -1 here stands for \"argument not given\", since None has meaning\n        # as a branch\n        def thd(conn):\n            sch_ch_tbl = self.db.model.scheduler_changes\n            ch_tbl = self.db.model.changes\n\n            wc = (sch_ch_tbl.c.schedulerid == schedulerid)\n\n            # may need to filter further based on branch, etc\n            extra_wheres = []\n            if branch != -1:\n                extra_wheres.append(ch_tbl.c.branch == branch)\n            if repository != -1:\n                extra_wheres.append(ch_tbl.c.repository == repository)\n            if project != -1:\n                extra_wheres.append(ch_tbl.c.project == project)\n            if codebase != -1:\n                extra_wheres.append(ch_tbl.c.codebase == codebase)\n\n            # if we need to filter further append those, as well as a join\n            # on changeid (but just once for that one)\n            if extra_wheres:\n                wc &= (sch_ch_tbl.c.changeid == ch_tbl.c.changeid)\n                for w in extra_wheres:\n                    wc &= w\n\n            q = sa.select(\n                [sch_ch_tbl.c.changeid, sch_ch_tbl.c.important],\n                whereclause=wc)\n            return dict([(r.changeid, [False, True][r.important])\n                         for r in conn.execute(q)])\n        return self.db.pool.do(thd)\n\n    def findSchedulerId(self, name):\n        tbl = self.db.model.schedulers\n        name_hash = self.hashColumns(name)\n        return self.findSomethingId(\n            tbl=tbl,\n            whereclause=(tbl.c.name_hash == name_hash),\n            insert_values=dict(\n                name=name,\n                name_hash=name_hash,\n            ))\n\n    def setSchedulerMaster(self, schedulerid, masterid):\n        def thd(conn):\n            sch_mst_tbl = self.db.model.scheduler_masters\n\n            # handle the masterid=None case to get it out of the way\n            if masterid is None:\n                q = sch_mst_tbl.delete(\n                    whereclause=(sch_mst_tbl.c.schedulerid == schedulerid))\n                conn.execute(q)\n                return\n\n            # try a blind insert..\n            try:\n                q = sch_mst_tbl.insert()\n                conn.execute(q,\n                             dict(schedulerid=schedulerid, masterid=masterid))\n            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):\n                # someone already owns this scheduler.\n                raise SchedulerAlreadyClaimedError\n\n        return self.db.pool.do(thd)\n\n    @defer.inlineCallbacks\n    def getScheduler(self, schedulerid):\n        sch = yield self.getSchedulers(_schedulerid=schedulerid)\n        if sch:\n            defer.returnValue(sch[0])\n\n    def getSchedulers(self, active=None, masterid=None, _schedulerid=None):\n        def thd(conn):\n            sch_tbl = self.db.model.schedulers\n            sch_mst_tbl = self.db.model.scheduler_masters\n\n            # handle the trivial case of masterid=xx and active=False\n            if masterid is not None and active is not None and not active:\n                return []\n\n            join = sch_tbl.outerjoin(sch_mst_tbl,\n                                     (sch_tbl.c.id == sch_mst_tbl.c.schedulerid))\n\n            # if we're given a _schedulerid, select only that row\n            wc = None\n            if _schedulerid:\n                wc = (sch_tbl.c.id == _schedulerid)\n            else:\n                # otherwise, filter with active, if necessary\n                if masterid is not None:\n                    wc = (sch_mst_tbl.c.masterid == masterid)\n                elif active:\n                    wc = (sch_mst_tbl.c.masterid != NULL)\n                elif active is not None:\n                    wc = (sch_mst_tbl.c.masterid == NULL)\n\n            q = sa.select([sch_tbl.c.id, sch_tbl.c.name,\n                           sch_mst_tbl.c.masterid],\n                          from_obj=join, whereclause=wc)\n\n            return [dict(id=row.id, name=row.name,\n                         masterid=row.masterid)\n                    for row in conn.execute(q).fetchall()]\n        return self.db.pool.do(thd)\n", "path": "master/buildbot/db/schedulers.py"}, {"content": "# This file is part of Buildbot.  Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nimport hashlib\nimport sqlalchemy as sa\n\nfrom migrate import changeset\n\n\ndef add_new_schema_parts(migrate_engine):\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n    sa.Table('builders', metadata, autoload=True)\n    sa.Table('buildsets', metadata, autoload=True)\n\n    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n\n    builderid = sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'))\n    builderid.create(buildrequests)\n\n    # Remove all index\n    for index in buildrequests.indexes:\n        index.drop()\n\n\ndef migrate_data(migrate_engine):\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n\n    # set up the tables we'll need to migrate\n    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n    builders = sa.Table('builders', metadata, autoload=True)\n\n    bName2bID = dict()\n    q = sa.select([builders.c.id, builders.c.name])\n    for row in migrate_engine.execute(q).fetchall():\n        bName2bID[row.name] = row.id\n\n    def hashColumns(*args):\n        # copy paste from buildbot/db/base.py\n        def encode(x):\n            try:\n                return x.encode('utf8')\n            except AttributeError:\n                if x is None:\n                    return '\\xf5'\n                return str(x)\n        return hashlib.sha1('\\0'.join(map(encode, args))).hexdigest()\n\n    def findbuilderid(buildername):\n        bid = bName2bID.get(buildername)\n        if bid is None:\n            r = migrate_engine.execute(builders.insert(), [{\n                'name': buildername,\n                'name_hash': hashColumns(buildername),\n            }])\n            bid = r.inserted_primary_key[0]\n            bName2bID[buildername] = bid\n        return bid\n\n    c = buildrequests.c\n    q = sa.select([c.id, c.buildername])\n    for row in migrate_engine.execute(q).fetchall():\n        builderid = findbuilderid(row.buildername)\n        migrate_engine.execute(\n            buildrequests.update(whereclause=(c.id == row.id)),\n            builderid=builderid)\n\n\ndef remove_buildername(migrate_engine):\n\n    metadata = sa.MetaData()\n    metadata.bind = migrate_engine\n\n    sa.Table('builders', metadata, autoload=True)\n    sa.Table('buildsets', metadata, autoload=True)\n    buildrequests = sa.Table('buildrequests', metadata, autoload=True)\n\n    # Specify what the new table should look like\n    buildrequests.c.buildername.drop()\n\n    changeset.alter_column(\n        sa.Column('builderid', sa.Integer, sa.ForeignKey(\"builders.id\"), nullable=False),\n        table=buildrequests,\n        metadata=metadata,\n        engine=migrate_engine)\n\n    idx = sa.Index('buildrequests_builderid', buildrequests.c.builderid)\n    idx.create(migrate_engine)\n    idx = sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid)\n    idx.create(migrate_engine)\n    idx = sa.Index('buildrequests_complete', buildrequests.c.complete)\n    idx.create(migrate_engine)\n\n\ndef upgrade(migrate_engine):\n    # add a 'builderid' column to buildrequests\n    add_new_schema_parts(migrate_engine)\n    # migrate the data to new tables\n    migrate_data(migrate_engine)\n    # Finally remove the buildername column\n    remove_buildername(migrate_engine)\n", "path": "master/buildbot/db/migrate/versions/037_buildrequests_builderid.py"}]}