We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
DockerLatentWorker: pull specified image if it is not yet on the system
This is merely an enhancement, but it would be nice if the specified image is pulled from dockerhub if it is not on yet on the host system. 
This should be pretty easy to accomplish using docker-py, but I don't know if this a desired feature.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `master/buildbot/worker/docker.py`
Content:
```
1 # This file is part of Buildbot. Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Copyright Buildbot Team Members
15 
16 from __future__ import absolute_import
17 from __future__ import division
18 from __future__ import print_function
19 
20 import hashlib
21 import json
22 import socket
23 from io import BytesIO
24 
25 from twisted.internet import defer
26 from twisted.internet import threads
27 from twisted.python import log
28 
29 from buildbot import config
30 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
31 from buildbot.util import unicode2bytes
32 from buildbot.worker import AbstractLatentWorker
33 
34 try:
35     import docker
36     from docker import client
37     from docker.errors import NotFound
38     _hush_pyflakes = [docker, client]
39 except ImportError:
40     docker = None
41     client = None
42 
43 
44 def _handle_stream_line(line):
45     """\
46     Input is the json representation of: {'stream': "Content\ncontent"}
47     Output is a generator yield "Content", and then "content"
48     """
49     # XXX This necessary processing is probably a bug from docker-py,
50     # hence, might break if the bug is fixed, i.e. we should get decoded JSON
51     # directly from the API.
52     line = json.loads(line)
53     if 'error' in line:
54         content = "ERROR: " + line['error']
55     else:
56         content = line.get('stream', '')
57     for streamline in content.split('\n'):
58         if streamline:
59             yield streamline
60 
61 
62 class DockerBaseWorker(AbstractLatentWorker):
63 
64     def checkConfig(self, name, password=None, image=None,
65                     masterFQDN=None, **kwargs):
66 
67         # Set build_wait_timeout to 0 if not explicitly set: Starting a
68         # container is almost immediate, we can afford doing so for each build.
69         if 'build_wait_timeout' not in kwargs:
70             kwargs['build_wait_timeout'] = 0
71         if image is not None and not isinstance(image, str):
72             if not hasattr(image, 'getRenderingFor'):
73                 config.error("image must be a string")
74 
75         AbstractLatentWorker.checkConfig(self, name, password, **kwargs)
76 
77     def reconfigService(self, name, password=None, image=None,
78                         masterFQDN=None, **kwargs):
79         # Set build_wait_timeout to 0 if not explicitly set: Starting a
80         # container is almost immediate, we can afford doing so for each build.
81         if 'build_wait_timeout' not in kwargs:
82             kwargs['build_wait_timeout'] = 0
83         if password is None:
84             password = self.getRandomPass()
85         if masterFQDN is None:
86             masterFQDN = socket.getfqdn()
87         self.masterFQDN = masterFQDN
88         self.image = image
89         masterName = unicode2bytes(self.master.name)
90         self.masterhash = hashlib.sha1(masterName).hexdigest()[:6]
91         return AbstractLatentWorker.reconfigService(self, name, password, **kwargs)
92 
93     def getContainerName(self):
94         return ('%s-%s' % ('buildbot' + self.masterhash, self.workername)).replace("_", "-")
95 
96     @property
97     def shortid(self):
98         if self.instance is None:
99             return None
100         return self.instance['Id'][:6]
101 
102     def createEnvironment(self):
103         result = {
104             "BUILDMASTER": self.masterFQDN,
105             "WORKERNAME": self.name,
106             "WORKERPASS": self.password
107         }
108         if self.registration is not None:
109             result["BUILDMASTER_PORT"] = str(self.registration.getPBPort())
110         if ":" in self.masterFQDN:
111             result["BUILDMASTER"], result["BUILDMASTER_PORT"] = self.masterFQDN.split(":")
112         return result
113 
114 
115 class DockerLatentWorker(DockerBaseWorker):
116     instance = None
117 
118     def checkConfig(self, name, password, docker_host, image=None, command=None,
119                     volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,
120                     masterFQDN=None, hostconfig=None, **kwargs):
121 
122         DockerBaseWorker.checkConfig(self, name, password, image, masterFQDN, **kwargs)
123 
124         if not client:
125             config.error("The python module 'docker-py>=1.4' is needed to use a"
126                          " DockerLatentWorker")
127         if not image and not dockerfile:
128             config.error("DockerLatentWorker: You need to specify at least"
129                          " an image name, or a dockerfile")
130 
131         # Following block is only for checking config errors,
132         # actual parsing happens in self.parse_volumes()
133         # Renderables can be direct volumes definition or list member
134         if isinstance(volumes, list):
135             for volume_string in (volumes or []):
136                 if not isinstance(volume_string, str):
137                     continue
138                 try:
139                     bind, volume = volume_string.split(":", 1)
140                 except ValueError:
141                     config.error("Invalid volume definition for docker "
142                                  "%s. Skipping..." % volume_string)
143                     continue
144 
145     @defer.inlineCallbacks
146     def reconfigService(self, name, password, docker_host, image=None, command=None,
147                         volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,
148                         masterFQDN=None, hostconfig=None, **kwargs):
149 
150         yield DockerBaseWorker.reconfigService(self, name, password, image, masterFQDN, **kwargs)
151         self.volumes = volumes or []
152         self.followStartupLogs = followStartupLogs
153 
154         self.command = command or []
155         self.dockerfile = dockerfile
156         self.hostconfig = hostconfig or {}
157         # Prepare the parameters for the Docker Client object.
158         self.client_args = {'base_url': docker_host}
159         if version is not None:
160             self.client_args['version'] = version
161         if tls is not None:
162             self.client_args['tls'] = tls
163 
164     def _thd_parse_volumes(self, volumes):
165         volume_list = []
166         binds = {}
167         for volume_string in (volumes or []):
168             try:
169                 bind, volume = volume_string.split(":", 1)
170             except ValueError:
171                 config.error("Invalid volume definition for docker "
172                              "%s. Skipping..." % volume_string)
173                 continue
174 
175             ro = False
176             if volume.endswith(':ro') or volume.endswith(':rw'):
177                 ro = volume[-2:] == 'ro'
178                 volume = volume[:-3]
179 
180             volume_list.append(volume)
181             binds[bind] = {'bind': volume, 'ro': ro}
182         return volume_list, binds
183 
184     def _getDockerClient(self):
185         if docker.version[0] == '1':
186             docker_client = client.Client(**self.client_args)
187         else:
188             docker_client = client.APIClient(**self.client_args)
189         return docker_client
190 
191     @defer.inlineCallbacks
192     def start_instance(self, build):
193         if self.instance is not None:
194             raise ValueError('instance active')
195         image = yield build.render(self.image)
196         dockerfile = yield build.render(self.dockerfile)
197         volumes = yield build.render(self.volumes)
198         res = yield threads.deferToThread(self._thd_start_instance, image, dockerfile, volumes)
199         defer.returnValue(res)
200 
201     def _image_exists(self, client, name):
202         # Make sure the image exists
203         for image in client.images():
204             for tag in image['RepoTags'] or []:
205                 if ':' in name and tag == name:
206                     return True
207                 if tag.startswith(name + ':'):
208                     return True
209         return False
210 
211     def _thd_start_instance(self, image, dockerfile, volumes):
212         docker_client = self._getDockerClient()
213         # cleanup the old instances
214         instances = docker_client.containers(
215             all=1,
216             filters=dict(name=self.getContainerName()))
217         for instance in instances:
218             try:
219                 docker_client.remove_container(instance['Id'], v=True, force=True)
220             except NotFound:
221                 pass  # that's a race condition
222 
223         found = False
224         if image is not None:
225             found = self._image_exists(docker_client, image)
226         else:
227             image = '%s_%s_image' % (self.workername, id(self))
228         if (not found) and (dockerfile is not None):
229             log.msg("Image '%s' not found, building it from scratch" %
230                     image)
231             for line in docker_client.build(fileobj=BytesIO(dockerfile.encode('utf-8')),
232                                             tag=image):
233                 for streamline in _handle_stream_line(line):
234                     log.msg(streamline)
235 
236         if (not self._image_exists(docker_client, image)):
237             log.msg("Image '%s' not found" % image)
238             raise LatentWorkerFailedToSubstantiate(
239                 'Image "%s" not found on docker host.' % image
240             )
241 
242         volumes, binds = self._thd_parse_volumes(volumes)
243         host_conf = self.hostconfig.copy()
244         host_conf['binds'] = binds
245         host_conf = docker_client.create_host_config(**host_conf)
246 
247         instance = docker_client.create_container(
248             image,
249             self.command,
250             name=self.getContainerName(),
251             volumes=volumes,
252             environment=self.createEnvironment(),
253             host_config=host_conf
254         )
255 
256         if instance.get('Id') is None:
257             log.msg('Failed to create the container')
258             raise LatentWorkerFailedToSubstantiate(
259                 'Failed to start container'
260             )
261         shortid = instance['Id'][:6]
262         log.msg('Container created, Id: %s...' % (shortid,))
263         instance['image'] = image
264         self.instance = instance
265         docker_client.start(instance)
266         log.msg('Container started')
267         if self.followStartupLogs:
268             logs = docker_client.attach(
269                 container=instance, stdout=True, stderr=True, stream=True)
270             for line in logs:
271                 log.msg("docker VM %s: %s" % (shortid, line.strip()))
272                 if self.conn:
273                     break
274             del logs
275         return [instance['Id'], image]
276 
277     def stop_instance(self, fast=False):
278         if self.instance is None:
279             # be gentle. Something may just be trying to alert us that an
280             # instance never attached, and it's because, somehow, we never
281             # started.
282             return defer.succeed(None)
283         instance = self.instance
284         self.instance = None
285         return threads.deferToThread(self._thd_stop_instance, instance, fast)
286 
287     def _thd_stop_instance(self, instance, fast):
288         docker_client = self._getDockerClient()
289         log.msg('Stopping container %s...' % instance['Id'][:6])
290         docker_client.stop(instance['Id'])
291         if not fast:
292             docker_client.wait(instance['Id'])
293         docker_client.remove_container(instance['Id'], v=True, force=True)
294         if self.image is None:
295             try:
296                 docker_client.remove_image(image=instance['image'])
297             except docker.errors.APIError as e:
298                 log.msg('Error while removing the image: %s', e)
299
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/master/buildbot/worker/docker.py b/master/buildbot/worker/docker.py
--- a/master/buildbot/worker/docker.py
+++ b/master/buildbot/worker/docker.py
@@ -117,7 +117,7 @@
 
     def checkConfig(self, name, password, docker_host, image=None, command=None,
                     volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,
-                    masterFQDN=None, hostconfig=None, **kwargs):
+                    masterFQDN=None, hostconfig=None, autopull=False, **kwargs):
 
         DockerBaseWorker.checkConfig(self, name, password, image, masterFQDN, **kwargs)
 
@@ -145,7 +145,7 @@
     @defer.inlineCallbacks
     def reconfigService(self, name, password, docker_host, image=None, command=None,
                         volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,
-                        masterFQDN=None, hostconfig=None, **kwargs):
+                        masterFQDN=None, hostconfig=None, autopull=False, **kwargs):
 
         yield DockerBaseWorker.reconfigService(self, name, password, image, masterFQDN, **kwargs)
         self.volumes = volumes or []
@@ -154,6 +154,7 @@
         self.command = command or []
         self.dockerfile = dockerfile
         self.hostconfig = hostconfig or {}
+        self.autopull = autopull
         # Prepare the parameters for the Docker Client object.
         self.client_args = {'base_url': docker_host}
         if version is not None:
@@ -233,6 +234,11 @@
                 for streamline in _handle_stream_line(line):
                     log.msg(streamline)
 
+        if ((not self._image_exists(docker_client, image))) and self.autopull:
+            log.msg("Image '%s' not found, pulling from registry" %
+                    image)
+            docker_client.pull(image)
+
         if (not self._image_exists(docker_client, image)):
             log.msg("Image '%s' not found" % image)
             raise LatentWorkerFailedToSubstantiate(

{"golden_diff": "diff --git a/master/buildbot/worker/docker.py b/master/buildbot/worker/docker.py\n--- a/master/buildbot/worker/docker.py\n+++ b/master/buildbot/worker/docker.py\n@@ -117,7 +117,7 @@\n \n     def checkConfig(self, name, password, docker_host, image=None, command=None,\n                     volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n-                    masterFQDN=None, hostconfig=None, **kwargs):\n+                    masterFQDN=None, hostconfig=None, autopull=False, **kwargs):\n \n         DockerBaseWorker.checkConfig(self, name, password, image, masterFQDN, **kwargs)\n \n@@ -145,7 +145,7 @@\n     @defer.inlineCallbacks\n     def reconfigService(self, name, password, docker_host, image=None, command=None,\n                         volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n-                        masterFQDN=None, hostconfig=None, **kwargs):\n+                        masterFQDN=None, hostconfig=None, autopull=False, **kwargs):\n \n         yield DockerBaseWorker.reconfigService(self, name, password, image, masterFQDN, **kwargs)\n         self.volumes = volumes or []\n@@ -154,6 +154,7 @@\n         self.command = command or []\n         self.dockerfile = dockerfile\n         self.hostconfig = hostconfig or {}\n+        self.autopull = autopull\n         # Prepare the parameters for the Docker Client object.\n         self.client_args = {'base_url': docker_host}\n         if version is not None:\n@@ -233,6 +234,11 @@\n                 for streamline in _handle_stream_line(line):\n                     log.msg(streamline)\n \n+        if ((not self._image_exists(docker_client, image))) and self.autopull:\n+            log.msg(\"Image '%s' not found, pulling from registry\" %\n+                    image)\n+            docker_client.pull(image)\n+\n         if (not self._image_exists(docker_client, image)):\n             log.msg(\"Image '%s' not found\" % image)\n             raise LatentWorkerFailedToSubstantiate(\n", "issue": "DockerLatentWorker: pull specified image if it is not yet on the system\nThis is merely an enhancement, but it would be nice if the specified image is pulled from dockerhub if it is not on yet on the host system. \r\nThis should be pretty easy to accomplish using docker-py, but I don't know if this a desired feature.\n", "before_files": [{"content": "# This file is part of Buildbot. Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport hashlib\nimport json\nimport socket\nfrom io import BytesIO\n\nfrom twisted.internet import defer\nfrom twisted.internet import threads\nfrom twisted.python import log\n\nfrom buildbot import config\nfrom buildbot.interfaces import LatentWorkerFailedToSubstantiate\nfrom buildbot.util import unicode2bytes\nfrom buildbot.worker import AbstractLatentWorker\n\ntry:\n    import docker\n    from docker import client\n    from docker.errors import NotFound\n    _hush_pyflakes = [docker, client]\nexcept ImportError:\n    docker = None\n    client = None\n\n\ndef _handle_stream_line(line):\n    \"\"\"\\\n    Input is the json representation of: {'stream': \"Content\\ncontent\"}\n    Output is a generator yield \"Content\", and then \"content\"\n    \"\"\"\n    # XXX This necessary processing is probably a bug from docker-py,\n    # hence, might break if the bug is fixed, i.e. we should get decoded JSON\n    # directly from the API.\n    line = json.loads(line)\n    if 'error' in line:\n        content = \"ERROR: \" + line['error']\n    else:\n        content = line.get('stream', '')\n    for streamline in content.split('\\n'):\n        if streamline:\n            yield streamline\n\n\nclass DockerBaseWorker(AbstractLatentWorker):\n\n    def checkConfig(self, name, password=None, image=None,\n                    masterFQDN=None, **kwargs):\n\n        # Set build_wait_timeout to 0 if not explicitly set: Starting a\n        # container is almost immediate, we can afford doing so for each build.\n        if 'build_wait_timeout' not in kwargs:\n            kwargs['build_wait_timeout'] = 0\n        if image is not None and not isinstance(image, str):\n            if not hasattr(image, 'getRenderingFor'):\n                config.error(\"image must be a string\")\n\n        AbstractLatentWorker.checkConfig(self, name, password, **kwargs)\n\n    def reconfigService(self, name, password=None, image=None,\n                        masterFQDN=None, **kwargs):\n        # Set build_wait_timeout to 0 if not explicitly set: Starting a\n        # container is almost immediate, we can afford doing so for each build.\n        if 'build_wait_timeout' not in kwargs:\n            kwargs['build_wait_timeout'] = 0\n        if password is None:\n            password = self.getRandomPass()\n        if masterFQDN is None:\n            masterFQDN = socket.getfqdn()\n        self.masterFQDN = masterFQDN\n        self.image = image\n        masterName = unicode2bytes(self.master.name)\n        self.masterhash = hashlib.sha1(masterName).hexdigest()[:6]\n        return AbstractLatentWorker.reconfigService(self, name, password, **kwargs)\n\n    def getContainerName(self):\n        return ('%s-%s' % ('buildbot' + self.masterhash, self.workername)).replace(\"_\", \"-\")\n\n    @property\n    def shortid(self):\n        if self.instance is None:\n            return None\n        return self.instance['Id'][:6]\n\n    def createEnvironment(self):\n        result = {\n            \"BUILDMASTER\": self.masterFQDN,\n            \"WORKERNAME\": self.name,\n            \"WORKERPASS\": self.password\n        }\n        if self.registration is not None:\n            result[\"BUILDMASTER_PORT\"] = str(self.registration.getPBPort())\n        if \":\" in self.masterFQDN:\n            result[\"BUILDMASTER\"], result[\"BUILDMASTER_PORT\"] = self.masterFQDN.split(\":\")\n        return result\n\n\nclass DockerLatentWorker(DockerBaseWorker):\n    instance = None\n\n    def checkConfig(self, name, password, docker_host, image=None, command=None,\n                    volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n                    masterFQDN=None, hostconfig=None, **kwargs):\n\n        DockerBaseWorker.checkConfig(self, name, password, image, masterFQDN, **kwargs)\n\n        if not client:\n            config.error(\"The python module 'docker-py>=1.4' is needed to use a\"\n                         \" DockerLatentWorker\")\n        if not image and not dockerfile:\n            config.error(\"DockerLatentWorker: You need to specify at least\"\n                         \" an image name, or a dockerfile\")\n\n        # Following block is only for checking config errors,\n        # actual parsing happens in self.parse_volumes()\n        # Renderables can be direct volumes definition or list member\n        if isinstance(volumes, list):\n            for volume_string in (volumes or []):\n                if not isinstance(volume_string, str):\n                    continue\n                try:\n                    bind, volume = volume_string.split(\":\", 1)\n                except ValueError:\n                    config.error(\"Invalid volume definition for docker \"\n                                 \"%s. Skipping...\" % volume_string)\n                    continue\n\n    @defer.inlineCallbacks\n    def reconfigService(self, name, password, docker_host, image=None, command=None,\n                        volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n                        masterFQDN=None, hostconfig=None, **kwargs):\n\n        yield DockerBaseWorker.reconfigService(self, name, password, image, masterFQDN, **kwargs)\n        self.volumes = volumes or []\n        self.followStartupLogs = followStartupLogs\n\n        self.command = command or []\n        self.dockerfile = dockerfile\n        self.hostconfig = hostconfig or {}\n        # Prepare the parameters for the Docker Client object.\n        self.client_args = {'base_url': docker_host}\n        if version is not None:\n            self.client_args['version'] = version\n        if tls is not None:\n            self.client_args['tls'] = tls\n\n    def _thd_parse_volumes(self, volumes):\n        volume_list = []\n        binds = {}\n        for volume_string in (volumes or []):\n            try:\n                bind, volume = volume_string.split(\":\", 1)\n            except ValueError:\n                config.error(\"Invalid volume definition for docker \"\n                             \"%s. Skipping...\" % volume_string)\n                continue\n\n            ro = False\n            if volume.endswith(':ro') or volume.endswith(':rw'):\n                ro = volume[-2:] == 'ro'\n                volume = volume[:-3]\n\n            volume_list.append(volume)\n            binds[bind] = {'bind': volume, 'ro': ro}\n        return volume_list, binds\n\n    def _getDockerClient(self):\n        if docker.version[0] == '1':\n            docker_client = client.Client(**self.client_args)\n        else:\n            docker_client = client.APIClient(**self.client_args)\n        return docker_client\n\n    @defer.inlineCallbacks\n    def start_instance(self, build):\n        if self.instance is not None:\n            raise ValueError('instance active')\n        image = yield build.render(self.image)\n        dockerfile = yield build.render(self.dockerfile)\n        volumes = yield build.render(self.volumes)\n        res = yield threads.deferToThread(self._thd_start_instance, image, dockerfile, volumes)\n        defer.returnValue(res)\n\n    def _image_exists(self, client, name):\n        # Make sure the image exists\n        for image in client.images():\n            for tag in image['RepoTags'] or []:\n                if ':' in name and tag == name:\n                    return True\n                if tag.startswith(name + ':'):\n                    return True\n        return False\n\n    def _thd_start_instance(self, image, dockerfile, volumes):\n        docker_client = self._getDockerClient()\n        # cleanup the old instances\n        instances = docker_client.containers(\n            all=1,\n            filters=dict(name=self.getContainerName()))\n        for instance in instances:\n            try:\n                docker_client.remove_container(instance['Id'], v=True, force=True)\n            except NotFound:\n                pass  # that's a race condition\n\n        found = False\n        if image is not None:\n            found = self._image_exists(docker_client, image)\n        else:\n            image = '%s_%s_image' % (self.workername, id(self))\n        if (not found) and (dockerfile is not None):\n            log.msg(\"Image '%s' not found, building it from scratch\" %\n                    image)\n            for line in docker_client.build(fileobj=BytesIO(dockerfile.encode('utf-8')),\n                                            tag=image):\n                for streamline in _handle_stream_line(line):\n                    log.msg(streamline)\n\n        if (not self._image_exists(docker_client, image)):\n            log.msg(\"Image '%s' not found\" % image)\n            raise LatentWorkerFailedToSubstantiate(\n                'Image \"%s\" not found on docker host.' % image\n            )\n\n        volumes, binds = self._thd_parse_volumes(volumes)\n        host_conf = self.hostconfig.copy()\n        host_conf['binds'] = binds\n        host_conf = docker_client.create_host_config(**host_conf)\n\n        instance = docker_client.create_container(\n            image,\n            self.command,\n            name=self.getContainerName(),\n            volumes=volumes,\n            environment=self.createEnvironment(),\n            host_config=host_conf\n        )\n\n        if instance.get('Id') is None:\n            log.msg('Failed to create the container')\n            raise LatentWorkerFailedToSubstantiate(\n                'Failed to start container'\n            )\n        shortid = instance['Id'][:6]\n        log.msg('Container created, Id: %s...' % (shortid,))\n        instance['image'] = image\n        self.instance = instance\n        docker_client.start(instance)\n        log.msg('Container started')\n        if self.followStartupLogs:\n            logs = docker_client.attach(\n                container=instance, stdout=True, stderr=True, stream=True)\n            for line in logs:\n                log.msg(\"docker VM %s: %s\" % (shortid, line.strip()))\n                if self.conn:\n                    break\n            del logs\n        return [instance['Id'], image]\n\n    def stop_instance(self, fast=False):\n        if self.instance is None:\n            # be gentle. Something may just be trying to alert us that an\n            # instance never attached, and it's because, somehow, we never\n            # started.\n            return defer.succeed(None)\n        instance = self.instance\n        self.instance = None\n        return threads.deferToThread(self._thd_stop_instance, instance, fast)\n\n    def _thd_stop_instance(self, instance, fast):\n        docker_client = self._getDockerClient()\n        log.msg('Stopping container %s...' % instance['Id'][:6])\n        docker_client.stop(instance['Id'])\n        if not fast:\n            docker_client.wait(instance['Id'])\n        docker_client.remove_container(instance['Id'], v=True, force=True)\n        if self.image is None:\n            try:\n                docker_client.remove_image(image=instance['image'])\n            except docker.errors.APIError as e:\n                log.msg('Error while removing the image: %s', e)\n", "path": "master/buildbot/worker/docker.py"}], "after_files": [{"content": "# This file is part of Buildbot. Buildbot is free software: you can\n# redistribute it and/or modify it under the terms of the GNU General Public\n# License as published by the Free Software Foundation, version 2.\n#\n# This program is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS\n# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more\n# details.\n#\n# You should have received a copy of the GNU General Public License along with\n# this program; if not, write to the Free Software Foundation, Inc., 51\n# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Copyright Buildbot Team Members\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport hashlib\nimport json\nimport socket\nfrom io import BytesIO\n\nfrom twisted.internet import defer\nfrom twisted.internet import threads\nfrom twisted.python import log\n\nfrom buildbot import config\nfrom buildbot.interfaces import LatentWorkerFailedToSubstantiate\nfrom buildbot.util import unicode2bytes\nfrom buildbot.worker import AbstractLatentWorker\n\ntry:\n    import docker\n    from docker import client\n    from docker.errors import NotFound\n    _hush_pyflakes = [docker, client]\nexcept ImportError:\n    docker = None\n    client = None\n\n\ndef _handle_stream_line(line):\n    \"\"\"\\\n    Input is the json representation of: {'stream': \"Content\\ncontent\"}\n    Output is a generator yield \"Content\", and then \"content\"\n    \"\"\"\n    # XXX This necessary processing is probably a bug from docker-py,\n    # hence, might break if the bug is fixed, i.e. we should get decoded JSON\n    # directly from the API.\n    line = json.loads(line)\n    if 'error' in line:\n        content = \"ERROR: \" + line['error']\n    else:\n        content = line.get('stream', '')\n    for streamline in content.split('\\n'):\n        if streamline:\n            yield streamline\n\n\nclass DockerBaseWorker(AbstractLatentWorker):\n\n    def checkConfig(self, name, password=None, image=None,\n                    masterFQDN=None, **kwargs):\n\n        # Set build_wait_timeout to 0 if not explicitly set: Starting a\n        # container is almost immediate, we can afford doing so for each build.\n        if 'build_wait_timeout' not in kwargs:\n            kwargs['build_wait_timeout'] = 0\n        if image is not None and not isinstance(image, str):\n            if not hasattr(image, 'getRenderingFor'):\n                config.error(\"image must be a string\")\n\n        AbstractLatentWorker.checkConfig(self, name, password, **kwargs)\n\n    def reconfigService(self, name, password=None, image=None,\n                        masterFQDN=None, **kwargs):\n        # Set build_wait_timeout to 0 if not explicitly set: Starting a\n        # container is almost immediate, we can afford doing so for each build.\n        if 'build_wait_timeout' not in kwargs:\n            kwargs['build_wait_timeout'] = 0\n        if password is None:\n            password = self.getRandomPass()\n        if masterFQDN is None:\n            masterFQDN = socket.getfqdn()\n        self.masterFQDN = masterFQDN\n        self.image = image\n        masterName = unicode2bytes(self.master.name)\n        self.masterhash = hashlib.sha1(masterName).hexdigest()[:6]\n        return AbstractLatentWorker.reconfigService(self, name, password, **kwargs)\n\n    def getContainerName(self):\n        return ('%s-%s' % ('buildbot' + self.masterhash, self.workername)).replace(\"_\", \"-\")\n\n    @property\n    def shortid(self):\n        if self.instance is None:\n            return None\n        return self.instance['Id'][:6]\n\n    def createEnvironment(self):\n        result = {\n            \"BUILDMASTER\": self.masterFQDN,\n            \"WORKERNAME\": self.name,\n            \"WORKERPASS\": self.password\n        }\n        if self.registration is not None:\n            result[\"BUILDMASTER_PORT\"] = str(self.registration.getPBPort())\n        if \":\" in self.masterFQDN:\n            result[\"BUILDMASTER\"], result[\"BUILDMASTER_PORT\"] = self.masterFQDN.split(\":\")\n        return result\n\n\nclass DockerLatentWorker(DockerBaseWorker):\n    instance = None\n\n    def checkConfig(self, name, password, docker_host, image=None, command=None,\n                    volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n                    masterFQDN=None, hostconfig=None, autopull=False, **kwargs):\n\n        DockerBaseWorker.checkConfig(self, name, password, image, masterFQDN, **kwargs)\n\n        if not client:\n            config.error(\"The python module 'docker-py>=1.4' is needed to use a\"\n                         \" DockerLatentWorker\")\n        if not image and not dockerfile:\n            config.error(\"DockerLatentWorker: You need to specify at least\"\n                         \" an image name, or a dockerfile\")\n\n        # Following block is only for checking config errors,\n        # actual parsing happens in self.parse_volumes()\n        # Renderables can be direct volumes definition or list member\n        if isinstance(volumes, list):\n            for volume_string in (volumes or []):\n                if not isinstance(volume_string, str):\n                    continue\n                try:\n                    bind, volume = volume_string.split(\":\", 1)\n                except ValueError:\n                    config.error(\"Invalid volume definition for docker \"\n                                 \"%s. Skipping...\" % volume_string)\n                    continue\n\n    @defer.inlineCallbacks\n    def reconfigService(self, name, password, docker_host, image=None, command=None,\n                        volumes=None, dockerfile=None, version=None, tls=None, followStartupLogs=False,\n                        masterFQDN=None, hostconfig=None, autopull=False, **kwargs):\n\n        yield DockerBaseWorker.reconfigService(self, name, password, image, masterFQDN, **kwargs)\n        self.volumes = volumes or []\n        self.followStartupLogs = followStartupLogs\n\n        self.command = command or []\n        self.dockerfile = dockerfile\n        self.hostconfig = hostconfig or {}\n        self.autopull = autopull\n        # Prepare the parameters for the Docker Client object.\n        self.client_args = {'base_url': docker_host}\n        if version is not None:\n            self.client_args['version'] = version\n        if tls is not None:\n            self.client_args['tls'] = tls\n\n    def _thd_parse_volumes(self, volumes):\n        volume_list = []\n        binds = {}\n        for volume_string in (volumes or []):\n            try:\n                bind, volume = volume_string.split(\":\", 1)\n            except ValueError:\n                config.error(\"Invalid volume definition for docker \"\n                             \"%s. Skipping...\" % volume_string)\n                continue\n\n            ro = False\n            if volume.endswith(':ro') or volume.endswith(':rw'):\n                ro = volume[-2:] == 'ro'\n                volume = volume[:-3]\n\n            volume_list.append(volume)\n            binds[bind] = {'bind': volume, 'ro': ro}\n        return volume_list, binds\n\n    def _getDockerClient(self):\n        if docker.version[0] == '1':\n            docker_client = client.Client(**self.client_args)\n        else:\n            docker_client = client.APIClient(**self.client_args)\n        return docker_client\n\n    @defer.inlineCallbacks\n    def start_instance(self, build):\n        if self.instance is not None:\n            raise ValueError('instance active')\n        image = yield build.render(self.image)\n        dockerfile = yield build.render(self.dockerfile)\n        volumes = yield build.render(self.volumes)\n        res = yield threads.deferToThread(self._thd_start_instance, image, dockerfile, volumes)\n        defer.returnValue(res)\n\n    def _image_exists(self, client, name):\n        # Make sure the image exists\n        for image in client.images():\n            for tag in image['RepoTags'] or []:\n                if ':' in name and tag == name:\n                    return True\n                if tag.startswith(name + ':'):\n                    return True\n        return False\n\n    def _thd_start_instance(self, image, dockerfile, volumes):\n        docker_client = self._getDockerClient()\n        # cleanup the old instances\n        instances = docker_client.containers(\n            all=1,\n            filters=dict(name=self.getContainerName()))\n        for instance in instances:\n            try:\n                docker_client.remove_container(instance['Id'], v=True, force=True)\n            except NotFound:\n                pass  # that's a race condition\n\n        found = False\n        if image is not None:\n            found = self._image_exists(docker_client, image)\n        else:\n            image = '%s_%s_image' % (self.workername, id(self))\n        if (not found) and (dockerfile is not None):\n            log.msg(\"Image '%s' not found, building it from scratch\" %\n                    image)\n            for line in docker_client.build(fileobj=BytesIO(dockerfile.encode('utf-8')),\n                                            tag=image):\n                for streamline in _handle_stream_line(line):\n                    log.msg(streamline)\n\n        if ((not self._image_exists(docker_client, image))) and self.autopull:\n            log.msg(\"Image '%s' not found, pulling from registry\" %\n                    image)\n            docker_client.pull(image)\n\n        if (not self._image_exists(docker_client, image)):\n            log.msg(\"Image '%s' not found\" % image)\n            raise LatentWorkerFailedToSubstantiate(\n                'Image \"%s\" not found on docker host.' % image\n            )\n\n        volumes, binds = self._thd_parse_volumes(volumes)\n        host_conf = self.hostconfig.copy()\n        host_conf['binds'] = binds\n        host_conf = docker_client.create_host_config(**host_conf)\n\n        instance = docker_client.create_container(\n            image,\n            self.command,\n            name=self.getContainerName(),\n            volumes=volumes,\n            environment=self.createEnvironment(),\n            host_config=host_conf\n        )\n\n        if instance.get('Id') is None:\n            log.msg('Failed to create the container')\n            raise LatentWorkerFailedToSubstantiate(\n                'Failed to start container'\n            )\n        shortid = instance['Id'][:6]\n        log.msg('Container created, Id: %s...' % (shortid,))\n        instance['image'] = image\n        self.instance = instance\n        docker_client.start(instance)\n        log.msg('Container started')\n        if self.followStartupLogs:\n            logs = docker_client.attach(\n                container=instance, stdout=True, stderr=True, stream=True)\n            for line in logs:\n                log.msg(\"docker VM %s: %s\" % (shortid, line.strip()))\n                if self.conn:\n                    break\n            del logs\n        return [instance['Id'], image]\n\n    def stop_instance(self, fast=False):\n        if self.instance is None:\n            # be gentle. Something may just be trying to alert us that an\n            # instance never attached, and it's because, somehow, we never\n            # started.\n            return defer.succeed(None)\n        instance = self.instance\n        self.instance = None\n        return threads.deferToThread(self._thd_stop_instance, instance, fast)\n\n    def _thd_stop_instance(self, instance, fast):\n        docker_client = self._getDockerClient()\n        log.msg('Stopping container %s...' % instance['Id'][:6])\n        docker_client.stop(instance['Id'])\n        if not fast:\n            docker_client.wait(instance['Id'])\n        docker_client.remove_container(instance['Id'], v=True, force=True)\n        if self.image is None:\n            try:\n                docker_client.remove_image(image=instance['image'])\n            except docker.errors.APIError as e:\n                log.msg('Error while removing the image: %s', e)\n", "path": "master/buildbot/worker/docker.py"}]}