We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Resource URL gets changed to ___
### CKAN Version if known (or site URL)
latest master

### The probem
When I create or edit a resource using the web form, whatever I set the resource.url to be, it gets saved as something like this: `http://myckansite.com/dataset/713c8c32-cbd8-46ce-90fb-86de1f0811d2/resource/59c0e90c-11e2-4d7f-af43-e098660745dc/download/___` (!)
It shouldn't change it. And the URL it sets it to gives a 404.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `ckan/lib/uploader.py`
Content:
```
1 # encoding: utf-8
2 
3 import os
4 import cgi
5 import datetime
6 import logging
7 import magic
8 import mimetypes
9 
10 from werkzeug.datastructures import FileStorage as FlaskFileStorage
11 
12 import ckan.lib.munge as munge
13 import ckan.logic as logic
14 import ckan.plugins as plugins
15 from ckan.common import config
16 
17 ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)
18 MB = 1 << 20
19 
20 log = logging.getLogger(__name__)
21 
22 _storage_path = None
23 _max_resource_size = None
24 _max_image_size = None
25 
26 
27 def _copy_file(input_file, output_file, max_size):
28     input_file.seek(0)
29     current_size = 0
30     while True:
31         current_size = current_size + 1
32         # MB chunks
33         data = input_file.read(MB)
34 
35         if not data:
36             break
37         output_file.write(data)
38         if current_size > max_size:
39             raise logic.ValidationError({'upload': ['File upload too large']})
40 
41 
42 def _get_underlying_file(wrapper):
43     if isinstance(wrapper, FlaskFileStorage):
44         return wrapper.stream
45     return wrapper.file
46 
47 
48 def get_uploader(upload_to, old_filename=None):
49     '''Query IUploader plugins and return an uploader instance for general
50     files.'''
51     upload = None
52     for plugin in plugins.PluginImplementations(plugins.IUploader):
53         upload = plugin.get_uploader(upload_to, old_filename)
54 
55     # default uploader
56     if upload is None:
57         upload = Upload(upload_to, old_filename)
58 
59     return upload
60 
61 
62 def get_resource_uploader(data_dict):
63     '''Query IUploader plugins and return a resource uploader instance.'''
64     upload = None
65     for plugin in plugins.PluginImplementations(plugins.IUploader):
66         upload = plugin.get_resource_uploader(data_dict)
67 
68     # default uploader
69     if upload is None:
70         upload = ResourceUpload(data_dict)
71 
72     return upload
73 
74 
75 def get_storage_path():
76     '''Function to cache storage path'''
77     global _storage_path
78 
79     # None means it has not been set. False means not in config.
80     if _storage_path is None:
81         storage_path = config.get('ckan.storage_path')
82         if storage_path:
83             _storage_path = storage_path
84         else:
85             log.critical('''Please specify a ckan.storage_path in your config
86                          for your uploads''')
87             _storage_path = False
88 
89     return _storage_path
90 
91 
92 def get_max_image_size():
93     global _max_image_size
94     if _max_image_size is None:
95         _max_image_size = int(config.get('ckan.max_image_size', 2))
96     return _max_image_size
97 
98 
99 def get_max_resource_size():
100     global _max_resource_size
101     if _max_resource_size is None:
102         _max_resource_size = int(config.get('ckan.max_resource_size', 10))
103     return _max_resource_size
104 
105 
106 class Upload(object):
107     def __init__(self, object_type, old_filename=None):
108         ''' Setup upload by creating a subdirectory of the storage directory
109         of name object_type. old_filename is the name of the file in the url
110         field last time'''
111 
112         self.storage_path = None
113         self.filename = None
114         self.filepath = None
115         path = get_storage_path()
116         if not path:
117             return
118         self.storage_path = os.path.join(path, 'storage',
119                                          'uploads', object_type)
120         try:
121             os.makedirs(self.storage_path)
122         except OSError as e:
123             # errno 17 is file already exists
124             if e.errno != 17:
125                 raise
126         self.object_type = object_type
127         self.old_filename = old_filename
128         if old_filename:
129             self.old_filepath = os.path.join(self.storage_path, old_filename)
130 
131     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
132         ''' Manipulate data from the data_dict.  url_field is the name of the
133         field where the upload is going to be. file_field is name of the key
134         where the FieldStorage is kept (i.e the field where the file data
135         actually is). clear_field is the name of a boolean field which
136         requests the upload to be deleted.  This needs to be called before
137         it reaches any validators'''
138 
139         self.url = data_dict.get(url_field, '')
140         self.clear = data_dict.pop(clear_field, None)
141         self.file_field = file_field
142         self.upload_field_storage = data_dict.pop(file_field, None)
143 
144         if not self.storage_path:
145             return
146 
147         if isinstance(self.upload_field_storage, (ALLOWED_UPLOAD_TYPES,)):
148             if self.upload_field_storage.filename:
149                 self.filename = self.upload_field_storage.filename
150                 self.filename = str(datetime.datetime.utcnow()) + self.filename
151                 self.filename = munge.munge_filename_legacy(self.filename)
152                 self.filepath = os.path.join(self.storage_path, self.filename)
153                 data_dict[url_field] = self.filename
154                 self.upload_file = _get_underlying_file(
155                     self.upload_field_storage)
156                 self.tmp_filepath = self.filepath + '~'
157         # keep the file if there has been no change
158         elif self.old_filename and not self.old_filename.startswith('http'):
159             if not self.clear:
160                 data_dict[url_field] = self.old_filename
161             if self.clear and self.url == self.old_filename:
162                 data_dict[url_field] = ''
163 
164     def upload(self, max_size=2):
165         ''' Actually upload the file.
166         This should happen just before a commit but after the data has
167         been validated and flushed to the db. This is so we do not store
168         anything unless the request is actually good.
169         max_size is size in MB maximum of the file'''
170 
171         if self.filename:
172             with open(self.tmp_filepath, 'wb+') as output_file:
173                 try:
174                     _copy_file(self.upload_file, output_file, max_size)
175                 except logic.ValidationError:
176                     os.remove(self.tmp_filepath)
177                     raise
178                 finally:
179                     self.upload_file.close()
180             os.rename(self.tmp_filepath, self.filepath)
181             self.clear = True
182 
183         if (self.clear and self.old_filename
184                 and not self.old_filename.startswith('http')):
185             try:
186                 os.remove(self.old_filepath)
187             except OSError:
188                 pass
189 
190 
191 class ResourceUpload(object):
192     def __init__(self, resource):
193         path = get_storage_path()
194         config_mimetype_guess = config.get('ckan.mimetype_guess', 'file_ext')
195 
196         if not path:
197             self.storage_path = None
198             return
199         self.storage_path = os.path.join(path, 'resources')
200         try:
201             os.makedirs(self.storage_path)
202         except OSError as e:
203             # errno 17 is file already exists
204             if e.errno != 17:
205                 raise
206         self.filename = None
207         self.mimetype = None
208 
209         url = resource.get('url')
210 
211         upload_field_storage = resource.pop('upload', None)
212         self.clear = resource.pop('clear_upload', None)
213 
214         if url and config_mimetype_guess == 'file_ext':
215             self.mimetype = mimetypes.guess_type(url)[0]
216 
217         if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
218             self.filesize = 0  # bytes
219 
220             self.filename = upload_field_storage.filename
221             self.filename = munge.munge_filename(self.filename)
222             resource['url'] = self.filename
223             resource['url_type'] = 'upload'
224             resource['last_modified'] = datetime.datetime.utcnow()
225             self.upload_file = _get_underlying_file(upload_field_storage)
226             self.upload_file.seek(0, os.SEEK_END)
227             self.filesize = self.upload_file.tell()
228             # go back to the beginning of the file buffer
229             self.upload_file.seek(0, os.SEEK_SET)
230 
231             # check if the mimetype failed from guessing with the url
232             if not self.mimetype and config_mimetype_guess == 'file_ext':
233                 self.mimetype = mimetypes.guess_type(self.filename)[0]
234 
235             if not self.mimetype and config_mimetype_guess == 'file_contents':
236                 try:
237                     self.mimetype = magic.from_buffer(self.upload_file.read(),
238                                                       mime=True)
239                     self.upload_file.seek(0, os.SEEK_SET)
240                 except IOError as e:
241                     # Not that important if call above fails
242                     self.mimetype = None
243 
244         elif self.clear:
245             resource['url_type'] = ''
246 
247     def get_directory(self, id):
248         directory = os.path.join(self.storage_path,
249                                  id[0:3], id[3:6])
250         return directory
251 
252     def get_path(self, id):
253         directory = self.get_directory(id)
254         filepath = os.path.join(directory, id[6:])
255         return filepath
256 
257     def upload(self, id, max_size=10):
258         '''Actually upload the file.
259 
260         :returns: ``'file uploaded'`` if a new file was successfully uploaded
261             (whether it overwrote a previously uploaded file or not),
262             ``'file deleted'`` if an existing uploaded file was deleted,
263             or ``None`` if nothing changed
264         :rtype: ``string`` or ``None``
265 
266         '''
267         if not self.storage_path:
268             return
269 
270         # Get directory and filepath on the system
271         # where the file for this resource will be stored
272         directory = self.get_directory(id)
273         filepath = self.get_path(id)
274 
275         # If a filename has been provided (a file is being uploaded)
276         # we write it to the filepath (and overwrite it if it already
277         # exists). This way the uploaded file will always be stored
278         # in the same location
279         if self.filename:
280             try:
281                 os.makedirs(directory)
282             except OSError as e:
283                 # errno 17 is file already exists
284                 if e.errno != 17:
285                     raise
286             tmp_filepath = filepath + '~'
287             with open(tmp_filepath, 'wb+') as output_file:
288                 try:
289                     _copy_file(self.upload_file, output_file, max_size)
290                 except logic.ValidationError:
291                     os.remove(tmp_filepath)
292                     raise
293                 finally:
294                     self.upload_file.close()
295             os.rename(tmp_filepath, filepath)
296             return
297 
298         # The resource form only sets self.clear (via the input clear_upload)
299         # to True when an uploaded file is not replaced by another uploaded
300         # file, only if it is replaced by a link to file.
301         # If the uploaded file is replaced by a link, we should remove the
302         # previously uploaded file to clean up the file system.
303         if self.clear:
304             try:
305                 os.remove(filepath)
306             except OSError as e:
307                 pass
308
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -214,7 +214,8 @@
         if url and config_mimetype_guess == 'file_ext':
             self.mimetype = mimetypes.guess_type(url)[0]
 
-        if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
+        if bool(upload_field_storage) and \
+                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):
             self.filesize = 0  # bytes
 
             self.filename = upload_field_storage.filename

{"golden_diff": "diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py\n--- a/ckan/lib/uploader.py\n+++ b/ckan/lib/uploader.py\n@@ -214,7 +214,8 @@\n         if url and config_mimetype_guess == 'file_ext':\n             self.mimetype = mimetypes.guess_type(url)[0]\n \n-        if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n+        if bool(upload_field_storage) and \\\n+                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n             self.filesize = 0  # bytes\n \n             self.filename = upload_field_storage.filename\n", "issue": "Resource URL gets changed to ___\n### CKAN Version if known (or site URL)\r\nlatest master\r\n\r\n### The probem\r\nWhen I create or edit a resource using the web form, whatever I set the resource.url to be, it gets saved as something like this: `http://myckansite.com/dataset/713c8c32-cbd8-46ce-90fb-86de1f0811d2/resource/59c0e90c-11e2-4d7f-af43-e098660745dc/download/___` (!)\r\nIt shouldn't change it. And the URL it sets it to gives a 404.\n", "before_files": [{"content": "# encoding: utf-8\n\nimport os\nimport cgi\nimport datetime\nimport logging\nimport magic\nimport mimetypes\n\nfrom werkzeug.datastructures import FileStorage as FlaskFileStorage\n\nimport ckan.lib.munge as munge\nimport ckan.logic as logic\nimport ckan.plugins as plugins\nfrom ckan.common import config\n\nALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)\nMB = 1 << 20\n\nlog = logging.getLogger(__name__)\n\n_storage_path = None\n_max_resource_size = None\n_max_image_size = None\n\n\ndef _copy_file(input_file, output_file, max_size):\n    input_file.seek(0)\n    current_size = 0\n    while True:\n        current_size = current_size + 1\n        # MB chunks\n        data = input_file.read(MB)\n\n        if not data:\n            break\n        output_file.write(data)\n        if current_size > max_size:\n            raise logic.ValidationError({'upload': ['File upload too large']})\n\n\ndef _get_underlying_file(wrapper):\n    if isinstance(wrapper, FlaskFileStorage):\n        return wrapper.stream\n    return wrapper.file\n\n\ndef get_uploader(upload_to, old_filename=None):\n    '''Query IUploader plugins and return an uploader instance for general\n    files.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_uploader(upload_to, old_filename)\n\n    # default uploader\n    if upload is None:\n        upload = Upload(upload_to, old_filename)\n\n    return upload\n\n\ndef get_resource_uploader(data_dict):\n    '''Query IUploader plugins and return a resource uploader instance.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_resource_uploader(data_dict)\n\n    # default uploader\n    if upload is None:\n        upload = ResourceUpload(data_dict)\n\n    return upload\n\n\ndef get_storage_path():\n    '''Function to cache storage path'''\n    global _storage_path\n\n    # None means it has not been set. False means not in config.\n    if _storage_path is None:\n        storage_path = config.get('ckan.storage_path')\n        if storage_path:\n            _storage_path = storage_path\n        else:\n            log.critical('''Please specify a ckan.storage_path in your config\n                         for your uploads''')\n            _storage_path = False\n\n    return _storage_path\n\n\ndef get_max_image_size():\n    global _max_image_size\n    if _max_image_size is None:\n        _max_image_size = int(config.get('ckan.max_image_size', 2))\n    return _max_image_size\n\n\ndef get_max_resource_size():\n    global _max_resource_size\n    if _max_resource_size is None:\n        _max_resource_size = int(config.get('ckan.max_resource_size', 10))\n    return _max_resource_size\n\n\nclass Upload(object):\n    def __init__(self, object_type, old_filename=None):\n        ''' Setup upload by creating a subdirectory of the storage directory\n        of name object_type. old_filename is the name of the file in the url\n        field last time'''\n\n        self.storage_path = None\n        self.filename = None\n        self.filepath = None\n        path = get_storage_path()\n        if not path:\n            return\n        self.storage_path = os.path.join(path, 'storage',\n                                         'uploads', object_type)\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.object_type = object_type\n        self.old_filename = old_filename\n        if old_filename:\n            self.old_filepath = os.path.join(self.storage_path, old_filename)\n\n    def update_data_dict(self, data_dict, url_field, file_field, clear_field):\n        ''' Manipulate data from the data_dict.  url_field is the name of the\n        field where the upload is going to be. file_field is name of the key\n        where the FieldStorage is kept (i.e the field where the file data\n        actually is). clear_field is the name of a boolean field which\n        requests the upload to be deleted.  This needs to be called before\n        it reaches any validators'''\n\n        self.url = data_dict.get(url_field, '')\n        self.clear = data_dict.pop(clear_field, None)\n        self.file_field = file_field\n        self.upload_field_storage = data_dict.pop(file_field, None)\n\n        if not self.storage_path:\n            return\n\n        if isinstance(self.upload_field_storage, (ALLOWED_UPLOAD_TYPES,)):\n            if self.upload_field_storage.filename:\n                self.filename = self.upload_field_storage.filename\n                self.filename = str(datetime.datetime.utcnow()) + self.filename\n                self.filename = munge.munge_filename_legacy(self.filename)\n                self.filepath = os.path.join(self.storage_path, self.filename)\n                data_dict[url_field] = self.filename\n                self.upload_file = _get_underlying_file(\n                    self.upload_field_storage)\n                self.tmp_filepath = self.filepath + '~'\n        # keep the file if there has been no change\n        elif self.old_filename and not self.old_filename.startswith('http'):\n            if not self.clear:\n                data_dict[url_field] = self.old_filename\n            if self.clear and self.url == self.old_filename:\n                data_dict[url_field] = ''\n\n    def upload(self, max_size=2):\n        ''' Actually upload the file.\n        This should happen just before a commit but after the data has\n        been validated and flushed to the db. This is so we do not store\n        anything unless the request is actually good.\n        max_size is size in MB maximum of the file'''\n\n        if self.filename:\n            with open(self.tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(self.tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(self.tmp_filepath, self.filepath)\n            self.clear = True\n\n        if (self.clear and self.old_filename\n                and not self.old_filename.startswith('http')):\n            try:\n                os.remove(self.old_filepath)\n            except OSError:\n                pass\n\n\nclass ResourceUpload(object):\n    def __init__(self, resource):\n        path = get_storage_path()\n        config_mimetype_guess = config.get('ckan.mimetype_guess', 'file_ext')\n\n        if not path:\n            self.storage_path = None\n            return\n        self.storage_path = os.path.join(path, 'resources')\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.filename = None\n        self.mimetype = None\n\n        url = resource.get('url')\n\n        upload_field_storage = resource.pop('upload', None)\n        self.clear = resource.pop('clear_upload', None)\n\n        if url and config_mimetype_guess == 'file_ext':\n            self.mimetype = mimetypes.guess_type(url)[0]\n\n        if isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            self.filesize = 0  # bytes\n\n            self.filename = upload_field_storage.filename\n            self.filename = munge.munge_filename(self.filename)\n            resource['url'] = self.filename\n            resource['url_type'] = 'upload'\n            resource['last_modified'] = datetime.datetime.utcnow()\n            self.upload_file = _get_underlying_file(upload_field_storage)\n            self.upload_file.seek(0, os.SEEK_END)\n            self.filesize = self.upload_file.tell()\n            # go back to the beginning of the file buffer\n            self.upload_file.seek(0, os.SEEK_SET)\n\n            # check if the mimetype failed from guessing with the url\n            if not self.mimetype and config_mimetype_guess == 'file_ext':\n                self.mimetype = mimetypes.guess_type(self.filename)[0]\n\n            if not self.mimetype and config_mimetype_guess == 'file_contents':\n                try:\n                    self.mimetype = magic.from_buffer(self.upload_file.read(),\n                                                      mime=True)\n                    self.upload_file.seek(0, os.SEEK_SET)\n                except IOError as e:\n                    # Not that important if call above fails\n                    self.mimetype = None\n\n        elif self.clear:\n            resource['url_type'] = ''\n\n    def get_directory(self, id):\n        directory = os.path.join(self.storage_path,\n                                 id[0:3], id[3:6])\n        return directory\n\n    def get_path(self, id):\n        directory = self.get_directory(id)\n        filepath = os.path.join(directory, id[6:])\n        return filepath\n\n    def upload(self, id, max_size=10):\n        '''Actually upload the file.\n\n        :returns: ``'file uploaded'`` if a new file was successfully uploaded\n            (whether it overwrote a previously uploaded file or not),\n            ``'file deleted'`` if an existing uploaded file was deleted,\n            or ``None`` if nothing changed\n        :rtype: ``string`` or ``None``\n\n        '''\n        if not self.storage_path:\n            return\n\n        # Get directory and filepath on the system\n        # where the file for this resource will be stored\n        directory = self.get_directory(id)\n        filepath = self.get_path(id)\n\n        # If a filename has been provided (a file is being uploaded)\n        # we write it to the filepath (and overwrite it if it already\n        # exists). This way the uploaded file will always be stored\n        # in the same location\n        if self.filename:\n            try:\n                os.makedirs(directory)\n            except OSError as e:\n                # errno 17 is file already exists\n                if e.errno != 17:\n                    raise\n            tmp_filepath = filepath + '~'\n            with open(tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(tmp_filepath, filepath)\n            return\n\n        # The resource form only sets self.clear (via the input clear_upload)\n        # to True when an uploaded file is not replaced by another uploaded\n        # file, only if it is replaced by a link to file.\n        # If the uploaded file is replaced by a link, we should remove the\n        # previously uploaded file to clean up the file system.\n        if self.clear:\n            try:\n                os.remove(filepath)\n            except OSError as e:\n                pass\n", "path": "ckan/lib/uploader.py"}], "after_files": [{"content": "# encoding: utf-8\n\nimport os\nimport cgi\nimport datetime\nimport logging\nimport magic\nimport mimetypes\n\nfrom werkzeug.datastructures import FileStorage as FlaskFileStorage\n\nimport ckan.lib.munge as munge\nimport ckan.logic as logic\nimport ckan.plugins as plugins\nfrom ckan.common import config\n\nALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage)\nMB = 1 << 20\n\nlog = logging.getLogger(__name__)\n\n_storage_path = None\n_max_resource_size = None\n_max_image_size = None\n\n\ndef _copy_file(input_file, output_file, max_size):\n    input_file.seek(0)\n    current_size = 0\n    while True:\n        current_size = current_size + 1\n        # MB chunks\n        data = input_file.read(MB)\n\n        if not data:\n            break\n        output_file.write(data)\n        if current_size > max_size:\n            raise logic.ValidationError({'upload': ['File upload too large']})\n\n\ndef _get_underlying_file(wrapper):\n    if isinstance(wrapper, FlaskFileStorage):\n        return wrapper.stream\n    return wrapper.file\n\n\ndef get_uploader(upload_to, old_filename=None):\n    '''Query IUploader plugins and return an uploader instance for general\n    files.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_uploader(upload_to, old_filename)\n\n    # default uploader\n    if upload is None:\n        upload = Upload(upload_to, old_filename)\n\n    return upload\n\n\ndef get_resource_uploader(data_dict):\n    '''Query IUploader plugins and return a resource uploader instance.'''\n    upload = None\n    for plugin in plugins.PluginImplementations(plugins.IUploader):\n        upload = plugin.get_resource_uploader(data_dict)\n\n    # default uploader\n    if upload is None:\n        upload = ResourceUpload(data_dict)\n\n    return upload\n\n\ndef get_storage_path():\n    '''Function to cache storage path'''\n    global _storage_path\n\n    # None means it has not been set. False means not in config.\n    if _storage_path is None:\n        storage_path = config.get('ckan.storage_path')\n        if storage_path:\n            _storage_path = storage_path\n        else:\n            log.critical('''Please specify a ckan.storage_path in your config\n                         for your uploads''')\n            _storage_path = False\n\n    return _storage_path\n\n\ndef get_max_image_size():\n    global _max_image_size\n    if _max_image_size is None:\n        _max_image_size = int(config.get('ckan.max_image_size', 2))\n    return _max_image_size\n\n\ndef get_max_resource_size():\n    global _max_resource_size\n    if _max_resource_size is None:\n        _max_resource_size = int(config.get('ckan.max_resource_size', 10))\n    return _max_resource_size\n\n\nclass Upload(object):\n    def __init__(self, object_type, old_filename=None):\n        ''' Setup upload by creating a subdirectory of the storage directory\n        of name object_type. old_filename is the name of the file in the url\n        field last time'''\n\n        self.storage_path = None\n        self.filename = None\n        self.filepath = None\n        path = get_storage_path()\n        if not path:\n            return\n        self.storage_path = os.path.join(path, 'storage',\n                                         'uploads', object_type)\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.object_type = object_type\n        self.old_filename = old_filename\n        if old_filename:\n            self.old_filepath = os.path.join(self.storage_path, old_filename)\n\n    def update_data_dict(self, data_dict, url_field, file_field, clear_field):\n        ''' Manipulate data from the data_dict.  url_field is the name of the\n        field where the upload is going to be. file_field is name of the key\n        where the FieldStorage is kept (i.e the field where the file data\n        actually is). clear_field is the name of a boolean field which\n        requests the upload to be deleted.  This needs to be called before\n        it reaches any validators'''\n\n        self.url = data_dict.get(url_field, '')\n        self.clear = data_dict.pop(clear_field, None)\n        self.file_field = file_field\n        self.upload_field_storage = data_dict.pop(file_field, None)\n\n        if not self.storage_path:\n            return\n\n        if isinstance(self.upload_field_storage, (ALLOWED_UPLOAD_TYPES,)):\n            if self.upload_field_storage.filename:\n                self.filename = self.upload_field_storage.filename\n                self.filename = str(datetime.datetime.utcnow()) + self.filename\n                self.filename = munge.munge_filename_legacy(self.filename)\n                self.filepath = os.path.join(self.storage_path, self.filename)\n                data_dict[url_field] = self.filename\n                self.upload_file = _get_underlying_file(\n                    self.upload_field_storage)\n                self.tmp_filepath = self.filepath + '~'\n        # keep the file if there has been no change\n        elif self.old_filename and not self.old_filename.startswith('http'):\n            if not self.clear:\n                data_dict[url_field] = self.old_filename\n            if self.clear and self.url == self.old_filename:\n                data_dict[url_field] = ''\n\n    def upload(self, max_size=2):\n        ''' Actually upload the file.\n        This should happen just before a commit but after the data has\n        been validated and flushed to the db. This is so we do not store\n        anything unless the request is actually good.\n        max_size is size in MB maximum of the file'''\n\n        if self.filename:\n            with open(self.tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(self.tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(self.tmp_filepath, self.filepath)\n            self.clear = True\n\n        if (self.clear and self.old_filename\n                and not self.old_filename.startswith('http')):\n            try:\n                os.remove(self.old_filepath)\n            except OSError:\n                pass\n\n\nclass ResourceUpload(object):\n    def __init__(self, resource):\n        path = get_storage_path()\n        config_mimetype_guess = config.get('ckan.mimetype_guess', 'file_ext')\n\n        if not path:\n            self.storage_path = None\n            return\n        self.storage_path = os.path.join(path, 'resources')\n        try:\n            os.makedirs(self.storage_path)\n        except OSError as e:\n            # errno 17 is file already exists\n            if e.errno != 17:\n                raise\n        self.filename = None\n        self.mimetype = None\n\n        url = resource.get('url')\n\n        upload_field_storage = resource.pop('upload', None)\n        self.clear = resource.pop('clear_upload', None)\n\n        if url and config_mimetype_guess == 'file_ext':\n            self.mimetype = mimetypes.guess_type(url)[0]\n\n        if bool(upload_field_storage) and \\\n                isinstance(upload_field_storage, ALLOWED_UPLOAD_TYPES):\n            self.filesize = 0  # bytes\n\n            self.filename = upload_field_storage.filename\n            self.filename = munge.munge_filename(self.filename)\n            resource['url'] = self.filename\n            resource['url_type'] = 'upload'\n            resource['last_modified'] = datetime.datetime.utcnow()\n            self.upload_file = _get_underlying_file(upload_field_storage)\n            self.upload_file.seek(0, os.SEEK_END)\n            self.filesize = self.upload_file.tell()\n            # go back to the beginning of the file buffer\n            self.upload_file.seek(0, os.SEEK_SET)\n\n            # check if the mimetype failed from guessing with the url\n            if not self.mimetype and config_mimetype_guess == 'file_ext':\n                self.mimetype = mimetypes.guess_type(self.filename)[0]\n\n            if not self.mimetype and config_mimetype_guess == 'file_contents':\n                try:\n                    self.mimetype = magic.from_buffer(self.upload_file.read(),\n                                                      mime=True)\n                    self.upload_file.seek(0, os.SEEK_SET)\n                except IOError as e:\n                    # Not that important if call above fails\n                    self.mimetype = None\n\n        elif self.clear:\n            resource['url_type'] = ''\n\n    def get_directory(self, id):\n        directory = os.path.join(self.storage_path,\n                                 id[0:3], id[3:6])\n        return directory\n\n    def get_path(self, id):\n        directory = self.get_directory(id)\n        filepath = os.path.join(directory, id[6:])\n        return filepath\n\n    def upload(self, id, max_size=10):\n        '''Actually upload the file.\n\n        :returns: ``'file uploaded'`` if a new file was successfully uploaded\n            (whether it overwrote a previously uploaded file or not),\n            ``'file deleted'`` if an existing uploaded file was deleted,\n            or ``None`` if nothing changed\n        :rtype: ``string`` or ``None``\n\n        '''\n        if not self.storage_path:\n            return\n\n        # Get directory and filepath on the system\n        # where the file for this resource will be stored\n        directory = self.get_directory(id)\n        filepath = self.get_path(id)\n\n        # If a filename has been provided (a file is being uploaded)\n        # we write it to the filepath (and overwrite it if it already\n        # exists). This way the uploaded file will always be stored\n        # in the same location\n        if self.filename:\n            try:\n                os.makedirs(directory)\n            except OSError as e:\n                # errno 17 is file already exists\n                if e.errno != 17:\n                    raise\n            tmp_filepath = filepath + '~'\n            with open(tmp_filepath, 'wb+') as output_file:\n                try:\n                    _copy_file(self.upload_file, output_file, max_size)\n                except logic.ValidationError:\n                    os.remove(tmp_filepath)\n                    raise\n                finally:\n                    self.upload_file.close()\n            os.rename(tmp_filepath, filepath)\n            return\n\n        # The resource form only sets self.clear (via the input clear_upload)\n        # to True when an uploaded file is not replaced by another uploaded\n        # file, only if it is replaced by a link to file.\n        # If the uploaded file is replaced by a link, we should remove the\n        # previously uploaded file to clean up the file system.\n        if self.clear:\n            try:\n                os.remove(filepath)\n            except OSError as e:\n                pass\n", "path": "ckan/lib/uploader.py"}]}