We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Storage patch is not thread-safe
This should only happen in development environments or other cases where MinIO is used in a Docker network.

```python
In [1]: from rgd.geodata.models import ChecksumFile
In [2]: from rgd.utility import patch_internal_presign
In [3]: f1 = ChecksumFile.objects.first()
In [4]: f2 = ChecksumFile.objects.first()
In [5]: with patch_internal_presign(f1.file):
   ...:     assert f1.file.storage.base_url != f2.file.storage.base_url
   ...:
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-5-37e9d3b557e8> in <module>
      1 with patch_internal_presign(f1.file):
----> 2     assert f1.file.storage.base_url != f2.file.storage.base_url
      3
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `django-rgd/rgd/utility.py`
Content:
```
1 import contextlib
2 from contextlib import contextmanager
3 from functools import wraps
4 import hashlib
5 import io
6 import logging
7 import os
8 from pathlib import Path
9 import shutil
10 import tempfile
11 from typing import Any, Generator
12 from urllib.error import HTTPError
13 from urllib.parse import urlparse
14 from urllib.request import urlopen
15 from uuid import uuid4
16 
17 import boto3
18 from botocore import UNSIGNED
19 from botocore.config import Config
20 from django.conf import settings
21 from django.contrib.gis.db.models import Model
22 from django.core.files import File
23 from django.db.models.fields.files import FieldFile
24 from django.utils.safestring import mark_safe
25 from filelock import FileLock, Timeout
26 import psutil
27 
28 try:
29     from minio_storage.storage import MinioStorage
30 except ImportError:
31     MinioStorage = None
32 
33 logger = logging.getLogger(__name__)
34 
35 
36 def get_temp_dir():
37     path = Path(getattr(settings, 'RGD_TEMP_DIR', os.path.join(tempfile.gettempdir(), 'rgd')))
38     path.mkdir(parents=True, exist_ok=True)
39     return path
40 
41 
42 def get_cache_dir():
43     path = Path(get_temp_dir(), 'file_cache')
44     path.mkdir(parents=True, exist_ok=True)
45     return path
46 
47 
48 def get_lock_dir():
49     path = Path(get_temp_dir(), 'file_locks')
50     path.mkdir(parents=True, exist_ok=True)
51     return path
52 
53 
54 def get_file_lock(path: Path):
55     """Create a file lock under the lock directory."""
56     # Computes the hash using Pathlib's hash implementation on absolute path
57     sha = hash(path.absolute())
58     lock_path = Path(get_lock_dir(), f'{sha}.lock')
59     lock = FileLock(lock_path)
60     return lock
61 
62 
63 @contextmanager
64 def safe_urlopen(url: str, *args, **kwargs):
65     with contextlib.closing(urlopen(url, *args, **kwargs)) as remote:
66         yield remote
67 
68 
69 def compute_hash(handle: io.BufferedIOBase, chunk_num_blocks: int = 128):
70     sha = hashlib.sha512()
71     while chunk := handle.read(chunk_num_blocks * sha.block_size):
72         sha.update(chunk)
73     return sha.hexdigest()
74 
75 
76 def compute_checksum_file_field(field_file: FieldFile, chunk_num_blocks: int = 128):
77     with field_file.open() as f:
78         return compute_hash(f, chunk_num_blocks)
79 
80 
81 def compute_checksum_url(url: str, chunk_num_blocks: int = 128):
82     with safe_urlopen(url) as remote:
83         return compute_hash(remote, chunk_num_blocks)
84 
85 
86 def _link_url(obj: Model, field: str):
87     if not getattr(obj, field, None):
88         return 'No attachment'
89     attr = getattr(obj, field)
90     if callable(attr):
91         url = attr()
92     else:
93         url = attr.url
94     return mark_safe(f'<a href="{url}" download>Download</a>')
95 
96 
97 def get_or_create_no_commit(model: Model, defaults: dict = None, **kwargs):
98     try:
99         return model.objects.get(**kwargs), False
100     except model.DoesNotExist:
101         if not defaults:
102             defaults = {}
103         defaults.update(kwargs)
104         return model(**defaults), True
105 
106 
107 def get_s3_client():
108     if boto3.session.Session().get_credentials():
109         s3 = boto3.client('s3')
110     else:
111         # No credentials present (often the case in dev), use unsigned requests
112         s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
113     return s3
114 
115 
116 def _download_url_file_to_stream(
117     url: str, dest_stream: io.BufferedIOBase, num_blocks: int = 128, block_size: int = 128
118 ):
119     parsed = urlparse(url)
120     if parsed.scheme == 's3':
121         s3 = get_s3_client()
122         s3.download_fileobj(parsed.netloc, parsed.path.lstrip('/'), dest_stream)
123     else:
124         with safe_urlopen(url) as remote:
125             while chunk := remote.read(num_blocks * block_size):
126                 dest_stream.write(chunk)
127                 dest_stream.flush()
128 
129 
130 def download_url_file_to_local_path(
131     url: str,
132     path: str,
133     num_blocks: int = 128,
134     block_size: int = 128,
135 ) -> Path:
136     dest_path = Path(path)
137     dest_path.parent.mkdir(parents=True, exist_ok=True)
138     with open(dest_path, 'wb') as dest_stream:
139         _download_url_file_to_stream(url, dest_stream, num_blocks=num_blocks, block_size=block_size)
140     return Path(dest_path)
141 
142 
143 def precheck_fuse(url: str) -> bool:
144     try:
145         import simple_httpfs  # noqa
146     except (ImportError, EnvironmentError):
147         return False
148     parsed = urlparse(url)
149     if parsed.scheme not in ['https', 'http']:
150         return False
151     try:
152         # The FUSE lib will not catch URL errors
153         with safe_urlopen(url) as _:
154             pass
155     except HTTPError:
156         return False
157     return True
158 
159 
160 @contextmanager
161 def url_file_to_fuse_path(url: str) -> Generator[Path, None, None]:
162     # Could raise ValueError within context
163     # Assumes `precheck_fuse` was verified prior
164     # See https://github.com/ResonantGeoData/ResonantGeoData/issues/237
165     parsed = urlparse(url)
166     if parsed.scheme == 'https':
167         fuse_path = url.replace('https://', '/tmp/rgd/https/') + '..'
168     elif parsed.scheme == 'http':
169         fuse_path = url.replace('http://', '/tmp/rgd/http/') + '..'
170     else:
171         raise ValueError(f'Scheme {parsed.scheme} not currently handled by FUSE.')
172     logger.info(f'FUSE path: {fuse_path}')
173     yield Path(fuse_path)
174 
175 
176 @contextmanager
177 def patch_internal_presign(f: FieldFile):
178     """Create an environment where Minio-based `FieldFile`s construct a locally accessible presigned URL.
179 
180     Sometimes the external host differs from the internal host for Minio files (e.g. in development).
181     Getting the URL in this context ensures that the presigned URL returns the correct host for the
182     odd situation of accessing the file locally.
183     """
184     if (
185         MinioStorage is not None
186         and isinstance(f.storage, MinioStorage)
187         and getattr(settings, 'MINIO_STORAGE_MEDIA_URL', None) is not None
188     ):
189         original_base_url = f.storage.base_url
190         try:
191             f.storage.base_url = None
192             yield
193         finally:
194             f.storage.base_url = original_base_url
195         return
196     yield
197 
198 
199 @contextmanager
200 def output_path_helper(filename: str, output: FieldFile):
201     workdir = get_temp_dir()
202     with tempfile.TemporaryDirectory(dir=workdir) as tmpdir:
203         output_path = os.path.join(tmpdir, filename)
204         try:
205             # Yield the path for the user to perform a task
206             yield output_path
207         except Exception as e:
208             raise e
209         else:
210             # Save the file contents to the output field only on success
211             with open(output_path, 'rb') as f:
212                 output.save(os.path.basename(output_path), f)
213 
214 
215 @contextmanager
216 def input_output_path_helper(source, output: FieldFile, prefix: str = '', suffix: str = ''):
217     """Yield source and output paths between a ChecksumFile and a FileFeild.
218 
219     The output path is saved to the output field after yielding.
220 
221     """
222     filename = prefix + os.path.basename(source.name) + suffix
223     with source.yield_local_path() as file_path:
224         filename = prefix + os.path.basename(source.name) + suffix
225         with output_path_helper(filename, output) as output_path:
226             try:
227                 # Yield the paths for the user to perform a task
228                 yield (file_path, output_path)
229             except Exception as e:
230                 raise e
231             else:
232                 # Save the file contents to the output field only on success
233                 with open(output_path, 'rb') as f:
234                     output.save(os.path.basename(output_path), f)
235 
236 
237 def uuid_prefix_filename(instance: Any, filename: str):
238     """Use a variable in settings to add a prefix to the path and keep the random uuid."""
239     prefix = getattr(settings, 'RGD_FILE_FIELD_PREFIX', None)
240     if prefix:
241         return f'{prefix}/{uuid4()}/{filename}'
242     return f'{uuid4()}/{filename}'
243 
244 
245 def skip_signal():
246     """Skip the signal on an instance-basis."""
247 
248     def _skip_signal(signal_func):
249         @wraps(signal_func)
250         def _decorator(sender, instance, *args, **kwargs):
251             if hasattr(instance, 'skip_signal') and instance.skip_signal:
252                 return None
253             return signal_func(sender, instance, *args, **kwargs)
254 
255         return _decorator
256 
257     return _skip_signal
258 
259 
260 def download_field_file_to_local_path(field_file: FieldFile, path: str) -> Path:
261     """Download entire FieldFile to disk location.
262 
263     This overrides `girder_utils.field_file_to_local_path` to download file to
264     local path without a context manager. Cleanup must be handled by caller.
265 
266     """
267     dest_path = Path(path)
268     dest_path.parent.mkdir(parents=True, exist_ok=True)
269     the_path = Path(dest_path)
270     with field_file.open('rb'):
271         file_obj: File = field_file.file
272         if type(file_obj) is File:
273             # When file_obj is an actual File, (typically backed by FileSystemStorage),
274             # it is already at a stable path on disk.
275             # We must symlink it into the desired path
276             os.symlink(file_obj.name, dest_path)
277             return the_path
278         else:
279             # When file_obj is actually a subclass of File, it only provides a Python
280             # file-like object API. So, it must be copied to a stable path.
281             with open(dest_path, 'wb') as dest_stream:
282                 shutil.copyfileobj(file_obj, dest_stream)
283                 dest_stream.flush()
284             return the_path
285 
286 
287 def clean_file_cache(override_target=None):
288     """Clean the file cache to achieve RGD_TARGET_AVAILABLE_CACHE (Gb).
289 
290     Return
291     ------
292     A tuple of the starting and ending free space in bytes.
293 
294     """
295     cache = get_cache_dir()
296     # Sort each directory by mtime
297     paths = sorted(Path(cache).iterdir(), key=os.path.getmtime)  # This sorts oldest to latest
298     # While free space is not enough, remove directories until all ar gone
299     initial = psutil.disk_usage(cache).free
300     target = override_target or getattr(settings, 'RGD_TARGET_AVAILABLE_CACHE', 2)
301     logger.info(f'Cleaning file cache... Starting free space is {initial} bytes.')
302     while psutil.disk_usage(cache).free * 1e-9 < target:
303         if not len(paths):
304             # If we delete everything and still cannot acheive target, warn
305             logger.error(
306                 f'Target cache free space of {target * 1e9} bytes not achieved when empty.'
307             )
308             break
309         path = paths.pop(0)
310         # Check if resource is locked and skip if so
311         lock = get_file_lock(path)
312         try:
313             with lock.acquire(timeout=0):
314                 if os.path.isdir(path):
315                     shutil.rmtree(path)
316                 else:
317                     os.remove(path)
318             # Remove the lockfile as well
319             os.remove(lock.lock_file)
320             logger.info(f'removed: {path}')
321         except Timeout:
322             # Another task holds the lock on this file/directory: do not delete
323             logger.info(f'File is locked, skipping: {path}')
324             pass
325     free = psutil.disk_usage(cache).free
326     logger.debug(f'Finished cleaning file cache. Available free space is {free} bytes.')
327     return initial, free
328 
329 
330 def purge_file_cache():
331     """Completely purge all files from the file cache.
332 
333     This should be used with extreme caution, it will delete files that could
334     be in use.
335 
336     """
337     cache = get_cache_dir()
338     shutil.rmtree(cache)
339     cache = get_cache_dir()  # Return the cache dir so that a fresh directory is created.
340     logger.debug(
341         f'Purged file cache. Available free space is {psutil.disk_usage(cache).free} bytes.'
342     )
343     return cache
344
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/django-rgd/rgd/utility.py b/django-rgd/rgd/utility.py
--- a/django-rgd/rgd/utility.py
+++ b/django-rgd/rgd/utility.py
@@ -180,6 +180,11 @@
     Sometimes the external host differs from the internal host for Minio files (e.g. in development).
     Getting the URL in this context ensures that the presigned URL returns the correct host for the
     odd situation of accessing the file locally.
+
+    Note
+    ----
+    If concerned regarding concurrent access, see https://github.com/ResonantGeoData/ResonantGeoData/issues/287
+
     """
     if (
         MinioStorage is not None

{"golden_diff": "diff --git a/django-rgd/rgd/utility.py b/django-rgd/rgd/utility.py\n--- a/django-rgd/rgd/utility.py\n+++ b/django-rgd/rgd/utility.py\n@@ -180,6 +180,11 @@\n     Sometimes the external host differs from the internal host for Minio files (e.g. in development).\n     Getting the URL in this context ensures that the presigned URL returns the correct host for the\n     odd situation of accessing the file locally.\n+\n+    Note\n+    ----\n+    If concerned regarding concurrent access, see https://github.com/ResonantGeoData/ResonantGeoData/issues/287\n+\n     \"\"\"\n     if (\n         MinioStorage is not None\n", "issue": "Storage patch is not thread-safe\nThis should only happen in development environments or other cases where MinIO is used in a Docker network.\r\n\r\n```python\r\nIn [1]: from rgd.geodata.models import ChecksumFile\r\nIn [2]: from rgd.utility import patch_internal_presign\r\nIn [3]: f1 = ChecksumFile.objects.first()\r\nIn [4]: f2 = ChecksumFile.objects.first()\r\nIn [5]: with patch_internal_presign(f1.file):\r\n   ...:     assert f1.file.storage.base_url != f2.file.storage.base_url\r\n   ...:\r\n---------------------------------------------------------------------------\r\nAssertionError                            Traceback (most recent call last)\r\n<ipython-input-5-37e9d3b557e8> in <module>\r\n      1 with patch_internal_presign(f1.file):\r\n----> 2     assert f1.file.storage.base_url != f2.file.storage.base_url\r\n      3\r\n```\n", "before_files": [{"content": "import contextlib\nfrom contextlib import contextmanager\nfrom functools import wraps\nimport hashlib\nimport io\nimport logging\nimport os\nfrom pathlib import Path\nimport shutil\nimport tempfile\nfrom typing import Any, Generator\nfrom urllib.error import HTTPError\nfrom urllib.parse import urlparse\nfrom urllib.request import urlopen\nfrom uuid import uuid4\n\nimport boto3\nfrom botocore import UNSIGNED\nfrom botocore.config import Config\nfrom django.conf import settings\nfrom django.contrib.gis.db.models import Model\nfrom django.core.files import File\nfrom django.db.models.fields.files import FieldFile\nfrom django.utils.safestring import mark_safe\nfrom filelock import FileLock, Timeout\nimport psutil\n\ntry:\n    from minio_storage.storage import MinioStorage\nexcept ImportError:\n    MinioStorage = None\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_temp_dir():\n    path = Path(getattr(settings, 'RGD_TEMP_DIR', os.path.join(tempfile.gettempdir(), 'rgd')))\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_cache_dir():\n    path = Path(get_temp_dir(), 'file_cache')\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_lock_dir():\n    path = Path(get_temp_dir(), 'file_locks')\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_file_lock(path: Path):\n    \"\"\"Create a file lock under the lock directory.\"\"\"\n    # Computes the hash using Pathlib's hash implementation on absolute path\n    sha = hash(path.absolute())\n    lock_path = Path(get_lock_dir(), f'{sha}.lock')\n    lock = FileLock(lock_path)\n    return lock\n\n\n@contextmanager\ndef safe_urlopen(url: str, *args, **kwargs):\n    with contextlib.closing(urlopen(url, *args, **kwargs)) as remote:\n        yield remote\n\n\ndef compute_hash(handle: io.BufferedIOBase, chunk_num_blocks: int = 128):\n    sha = hashlib.sha512()\n    while chunk := handle.read(chunk_num_blocks * sha.block_size):\n        sha.update(chunk)\n    return sha.hexdigest()\n\n\ndef compute_checksum_file_field(field_file: FieldFile, chunk_num_blocks: int = 128):\n    with field_file.open() as f:\n        return compute_hash(f, chunk_num_blocks)\n\n\ndef compute_checksum_url(url: str, chunk_num_blocks: int = 128):\n    with safe_urlopen(url) as remote:\n        return compute_hash(remote, chunk_num_blocks)\n\n\ndef _link_url(obj: Model, field: str):\n    if not getattr(obj, field, None):\n        return 'No attachment'\n    attr = getattr(obj, field)\n    if callable(attr):\n        url = attr()\n    else:\n        url = attr.url\n    return mark_safe(f'<a href=\"{url}\" download>Download</a>')\n\n\ndef get_or_create_no_commit(model: Model, defaults: dict = None, **kwargs):\n    try:\n        return model.objects.get(**kwargs), False\n    except model.DoesNotExist:\n        if not defaults:\n            defaults = {}\n        defaults.update(kwargs)\n        return model(**defaults), True\n\n\ndef get_s3_client():\n    if boto3.session.Session().get_credentials():\n        s3 = boto3.client('s3')\n    else:\n        # No credentials present (often the case in dev), use unsigned requests\n        s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))\n    return s3\n\n\ndef _download_url_file_to_stream(\n    url: str, dest_stream: io.BufferedIOBase, num_blocks: int = 128, block_size: int = 128\n):\n    parsed = urlparse(url)\n    if parsed.scheme == 's3':\n        s3 = get_s3_client()\n        s3.download_fileobj(parsed.netloc, parsed.path.lstrip('/'), dest_stream)\n    else:\n        with safe_urlopen(url) as remote:\n            while chunk := remote.read(num_blocks * block_size):\n                dest_stream.write(chunk)\n                dest_stream.flush()\n\n\ndef download_url_file_to_local_path(\n    url: str,\n    path: str,\n    num_blocks: int = 128,\n    block_size: int = 128,\n) -> Path:\n    dest_path = Path(path)\n    dest_path.parent.mkdir(parents=True, exist_ok=True)\n    with open(dest_path, 'wb') as dest_stream:\n        _download_url_file_to_stream(url, dest_stream, num_blocks=num_blocks, block_size=block_size)\n    return Path(dest_path)\n\n\ndef precheck_fuse(url: str) -> bool:\n    try:\n        import simple_httpfs  # noqa\n    except (ImportError, EnvironmentError):\n        return False\n    parsed = urlparse(url)\n    if parsed.scheme not in ['https', 'http']:\n        return False\n    try:\n        # The FUSE lib will not catch URL errors\n        with safe_urlopen(url) as _:\n            pass\n    except HTTPError:\n        return False\n    return True\n\n\n@contextmanager\ndef url_file_to_fuse_path(url: str) -> Generator[Path, None, None]:\n    # Could raise ValueError within context\n    # Assumes `precheck_fuse` was verified prior\n    # See https://github.com/ResonantGeoData/ResonantGeoData/issues/237\n    parsed = urlparse(url)\n    if parsed.scheme == 'https':\n        fuse_path = url.replace('https://', '/tmp/rgd/https/') + '..'\n    elif parsed.scheme == 'http':\n        fuse_path = url.replace('http://', '/tmp/rgd/http/') + '..'\n    else:\n        raise ValueError(f'Scheme {parsed.scheme} not currently handled by FUSE.')\n    logger.info(f'FUSE path: {fuse_path}')\n    yield Path(fuse_path)\n\n\n@contextmanager\ndef patch_internal_presign(f: FieldFile):\n    \"\"\"Create an environment where Minio-based `FieldFile`s construct a locally accessible presigned URL.\n\n    Sometimes the external host differs from the internal host for Minio files (e.g. in development).\n    Getting the URL in this context ensures that the presigned URL returns the correct host for the\n    odd situation of accessing the file locally.\n    \"\"\"\n    if (\n        MinioStorage is not None\n        and isinstance(f.storage, MinioStorage)\n        and getattr(settings, 'MINIO_STORAGE_MEDIA_URL', None) is not None\n    ):\n        original_base_url = f.storage.base_url\n        try:\n            f.storage.base_url = None\n            yield\n        finally:\n            f.storage.base_url = original_base_url\n        return\n    yield\n\n\n@contextmanager\ndef output_path_helper(filename: str, output: FieldFile):\n    workdir = get_temp_dir()\n    with tempfile.TemporaryDirectory(dir=workdir) as tmpdir:\n        output_path = os.path.join(tmpdir, filename)\n        try:\n            # Yield the path for the user to perform a task\n            yield output_path\n        except Exception as e:\n            raise e\n        else:\n            # Save the file contents to the output field only on success\n            with open(output_path, 'rb') as f:\n                output.save(os.path.basename(output_path), f)\n\n\n@contextmanager\ndef input_output_path_helper(source, output: FieldFile, prefix: str = '', suffix: str = ''):\n    \"\"\"Yield source and output paths between a ChecksumFile and a FileFeild.\n\n    The output path is saved to the output field after yielding.\n\n    \"\"\"\n    filename = prefix + os.path.basename(source.name) + suffix\n    with source.yield_local_path() as file_path:\n        filename = prefix + os.path.basename(source.name) + suffix\n        with output_path_helper(filename, output) as output_path:\n            try:\n                # Yield the paths for the user to perform a task\n                yield (file_path, output_path)\n            except Exception as e:\n                raise e\n            else:\n                # Save the file contents to the output field only on success\n                with open(output_path, 'rb') as f:\n                    output.save(os.path.basename(output_path), f)\n\n\ndef uuid_prefix_filename(instance: Any, filename: str):\n    \"\"\"Use a variable in settings to add a prefix to the path and keep the random uuid.\"\"\"\n    prefix = getattr(settings, 'RGD_FILE_FIELD_PREFIX', None)\n    if prefix:\n        return f'{prefix}/{uuid4()}/{filename}'\n    return f'{uuid4()}/{filename}'\n\n\ndef skip_signal():\n    \"\"\"Skip the signal on an instance-basis.\"\"\"\n\n    def _skip_signal(signal_func):\n        @wraps(signal_func)\n        def _decorator(sender, instance, *args, **kwargs):\n            if hasattr(instance, 'skip_signal') and instance.skip_signal:\n                return None\n            return signal_func(sender, instance, *args, **kwargs)\n\n        return _decorator\n\n    return _skip_signal\n\n\ndef download_field_file_to_local_path(field_file: FieldFile, path: str) -> Path:\n    \"\"\"Download entire FieldFile to disk location.\n\n    This overrides `girder_utils.field_file_to_local_path` to download file to\n    local path without a context manager. Cleanup must be handled by caller.\n\n    \"\"\"\n    dest_path = Path(path)\n    dest_path.parent.mkdir(parents=True, exist_ok=True)\n    the_path = Path(dest_path)\n    with field_file.open('rb'):\n        file_obj: File = field_file.file\n        if type(file_obj) is File:\n            # When file_obj is an actual File, (typically backed by FileSystemStorage),\n            # it is already at a stable path on disk.\n            # We must symlink it into the desired path\n            os.symlink(file_obj.name, dest_path)\n            return the_path\n        else:\n            # When file_obj is actually a subclass of File, it only provides a Python\n            # file-like object API. So, it must be copied to a stable path.\n            with open(dest_path, 'wb') as dest_stream:\n                shutil.copyfileobj(file_obj, dest_stream)\n                dest_stream.flush()\n            return the_path\n\n\ndef clean_file_cache(override_target=None):\n    \"\"\"Clean the file cache to achieve RGD_TARGET_AVAILABLE_CACHE (Gb).\n\n    Return\n    ------\n    A tuple of the starting and ending free space in bytes.\n\n    \"\"\"\n    cache = get_cache_dir()\n    # Sort each directory by mtime\n    paths = sorted(Path(cache).iterdir(), key=os.path.getmtime)  # This sorts oldest to latest\n    # While free space is not enough, remove directories until all ar gone\n    initial = psutil.disk_usage(cache).free\n    target = override_target or getattr(settings, 'RGD_TARGET_AVAILABLE_CACHE', 2)\n    logger.info(f'Cleaning file cache... Starting free space is {initial} bytes.')\n    while psutil.disk_usage(cache).free * 1e-9 < target:\n        if not len(paths):\n            # If we delete everything and still cannot acheive target, warn\n            logger.error(\n                f'Target cache free space of {target * 1e9} bytes not achieved when empty.'\n            )\n            break\n        path = paths.pop(0)\n        # Check if resource is locked and skip if so\n        lock = get_file_lock(path)\n        try:\n            with lock.acquire(timeout=0):\n                if os.path.isdir(path):\n                    shutil.rmtree(path)\n                else:\n                    os.remove(path)\n            # Remove the lockfile as well\n            os.remove(lock.lock_file)\n            logger.info(f'removed: {path}')\n        except Timeout:\n            # Another task holds the lock on this file/directory: do not delete\n            logger.info(f'File is locked, skipping: {path}')\n            pass\n    free = psutil.disk_usage(cache).free\n    logger.debug(f'Finished cleaning file cache. Available free space is {free} bytes.')\n    return initial, free\n\n\ndef purge_file_cache():\n    \"\"\"Completely purge all files from the file cache.\n\n    This should be used with extreme caution, it will delete files that could\n    be in use.\n\n    \"\"\"\n    cache = get_cache_dir()\n    shutil.rmtree(cache)\n    cache = get_cache_dir()  # Return the cache dir so that a fresh directory is created.\n    logger.debug(\n        f'Purged file cache. Available free space is {psutil.disk_usage(cache).free} bytes.'\n    )\n    return cache\n", "path": "django-rgd/rgd/utility.py"}], "after_files": [{"content": "import contextlib\nfrom contextlib import contextmanager\nfrom functools import wraps\nimport hashlib\nimport io\nimport logging\nimport os\nfrom pathlib import Path\nimport shutil\nimport tempfile\nfrom typing import Any, Generator\nfrom urllib.error import HTTPError\nfrom urllib.parse import urlparse\nfrom urllib.request import urlopen\nfrom uuid import uuid4\n\nimport boto3\nfrom botocore import UNSIGNED\nfrom botocore.config import Config\nfrom django.conf import settings\nfrom django.contrib.gis.db.models import Model\nfrom django.core.files import File\nfrom django.db.models.fields.files import FieldFile\nfrom django.utils.safestring import mark_safe\nfrom filelock import FileLock, Timeout\nimport psutil\n\ntry:\n    from minio_storage.storage import MinioStorage\nexcept ImportError:\n    MinioStorage = None\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_temp_dir():\n    path = Path(getattr(settings, 'RGD_TEMP_DIR', os.path.join(tempfile.gettempdir(), 'rgd')))\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_cache_dir():\n    path = Path(get_temp_dir(), 'file_cache')\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_lock_dir():\n    path = Path(get_temp_dir(), 'file_locks')\n    path.mkdir(parents=True, exist_ok=True)\n    return path\n\n\ndef get_file_lock(path: Path):\n    \"\"\"Create a file lock under the lock directory.\"\"\"\n    # Computes the hash using Pathlib's hash implementation on absolute path\n    sha = hash(path.absolute())\n    lock_path = Path(get_lock_dir(), f'{sha}.lock')\n    lock = FileLock(lock_path)\n    return lock\n\n\n@contextmanager\ndef safe_urlopen(url: str, *args, **kwargs):\n    with contextlib.closing(urlopen(url, *args, **kwargs)) as remote:\n        yield remote\n\n\ndef compute_hash(handle: io.BufferedIOBase, chunk_num_blocks: int = 128):\n    sha = hashlib.sha512()\n    while chunk := handle.read(chunk_num_blocks * sha.block_size):\n        sha.update(chunk)\n    return sha.hexdigest()\n\n\ndef compute_checksum_file_field(field_file: FieldFile, chunk_num_blocks: int = 128):\n    with field_file.open() as f:\n        return compute_hash(f, chunk_num_blocks)\n\n\ndef compute_checksum_url(url: str, chunk_num_blocks: int = 128):\n    with safe_urlopen(url) as remote:\n        return compute_hash(remote, chunk_num_blocks)\n\n\ndef _link_url(obj: Model, field: str):\n    if not getattr(obj, field, None):\n        return 'No attachment'\n    attr = getattr(obj, field)\n    if callable(attr):\n        url = attr()\n    else:\n        url = attr.url\n    return mark_safe(f'<a href=\"{url}\" download>Download</a>')\n\n\ndef get_or_create_no_commit(model: Model, defaults: dict = None, **kwargs):\n    try:\n        return model.objects.get(**kwargs), False\n    except model.DoesNotExist:\n        if not defaults:\n            defaults = {}\n        defaults.update(kwargs)\n        return model(**defaults), True\n\n\ndef get_s3_client():\n    if boto3.session.Session().get_credentials():\n        s3 = boto3.client('s3')\n    else:\n        # No credentials present (often the case in dev), use unsigned requests\n        s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))\n    return s3\n\n\ndef _download_url_file_to_stream(\n    url: str, dest_stream: io.BufferedIOBase, num_blocks: int = 128, block_size: int = 128\n):\n    parsed = urlparse(url)\n    if parsed.scheme == 's3':\n        s3 = get_s3_client()\n        s3.download_fileobj(parsed.netloc, parsed.path.lstrip('/'), dest_stream)\n    else:\n        with safe_urlopen(url) as remote:\n            while chunk := remote.read(num_blocks * block_size):\n                dest_stream.write(chunk)\n                dest_stream.flush()\n\n\ndef download_url_file_to_local_path(\n    url: str,\n    path: str,\n    num_blocks: int = 128,\n    block_size: int = 128,\n) -> Path:\n    dest_path = Path(path)\n    dest_path.parent.mkdir(parents=True, exist_ok=True)\n    with open(dest_path, 'wb') as dest_stream:\n        _download_url_file_to_stream(url, dest_stream, num_blocks=num_blocks, block_size=block_size)\n    return Path(dest_path)\n\n\ndef precheck_fuse(url: str) -> bool:\n    try:\n        import simple_httpfs  # noqa\n    except (ImportError, EnvironmentError):\n        return False\n    parsed = urlparse(url)\n    if parsed.scheme not in ['https', 'http']:\n        return False\n    try:\n        # The FUSE lib will not catch URL errors\n        with safe_urlopen(url) as _:\n            pass\n    except HTTPError:\n        return False\n    return True\n\n\n@contextmanager\ndef url_file_to_fuse_path(url: str) -> Generator[Path, None, None]:\n    # Could raise ValueError within context\n    # Assumes `precheck_fuse` was verified prior\n    # See https://github.com/ResonantGeoData/ResonantGeoData/issues/237\n    parsed = urlparse(url)\n    if parsed.scheme == 'https':\n        fuse_path = url.replace('https://', '/tmp/rgd/https/') + '..'\n    elif parsed.scheme == 'http':\n        fuse_path = url.replace('http://', '/tmp/rgd/http/') + '..'\n    else:\n        raise ValueError(f'Scheme {parsed.scheme} not currently handled by FUSE.')\n    logger.info(f'FUSE path: {fuse_path}')\n    yield Path(fuse_path)\n\n\n@contextmanager\ndef patch_internal_presign(f: FieldFile):\n    \"\"\"Create an environment where Minio-based `FieldFile`s construct a locally accessible presigned URL.\n\n    Sometimes the external host differs from the internal host for Minio files (e.g. in development).\n    Getting the URL in this context ensures that the presigned URL returns the correct host for the\n    odd situation of accessing the file locally.\n\n    Note\n    ----\n    If concerned regarding concurrent access, see https://github.com/ResonantGeoData/ResonantGeoData/issues/287\n\n    \"\"\"\n    if (\n        MinioStorage is not None\n        and isinstance(f.storage, MinioStorage)\n        and getattr(settings, 'MINIO_STORAGE_MEDIA_URL', None) is not None\n    ):\n        original_base_url = f.storage.base_url\n        try:\n            f.storage.base_url = None\n            yield\n        finally:\n            f.storage.base_url = original_base_url\n        return\n    yield\n\n\n@contextmanager\ndef output_path_helper(filename: str, output: FieldFile):\n    workdir = get_temp_dir()\n    with tempfile.TemporaryDirectory(dir=workdir) as tmpdir:\n        output_path = os.path.join(tmpdir, filename)\n        try:\n            # Yield the path for the user to perform a task\n            yield output_path\n        except Exception as e:\n            raise e\n        else:\n            # Save the file contents to the output field only on success\n            with open(output_path, 'rb') as f:\n                output.save(os.path.basename(output_path), f)\n\n\n@contextmanager\ndef input_output_path_helper(source, output: FieldFile, prefix: str = '', suffix: str = ''):\n    \"\"\"Yield source and output paths between a ChecksumFile and a FileFeild.\n\n    The output path is saved to the output field after yielding.\n\n    \"\"\"\n    filename = prefix + os.path.basename(source.name) + suffix\n    with source.yield_local_path() as file_path:\n        filename = prefix + os.path.basename(source.name) + suffix\n        with output_path_helper(filename, output) as output_path:\n            try:\n                # Yield the paths for the user to perform a task\n                yield (file_path, output_path)\n            except Exception as e:\n                raise e\n            else:\n                # Save the file contents to the output field only on success\n                with open(output_path, 'rb') as f:\n                    output.save(os.path.basename(output_path), f)\n\n\ndef uuid_prefix_filename(instance: Any, filename: str):\n    \"\"\"Use a variable in settings to add a prefix to the path and keep the random uuid.\"\"\"\n    prefix = getattr(settings, 'RGD_FILE_FIELD_PREFIX', None)\n    if prefix:\n        return f'{prefix}/{uuid4()}/{filename}'\n    return f'{uuid4()}/{filename}'\n\n\ndef skip_signal():\n    \"\"\"Skip the signal on an instance-basis.\"\"\"\n\n    def _skip_signal(signal_func):\n        @wraps(signal_func)\n        def _decorator(sender, instance, *args, **kwargs):\n            if hasattr(instance, 'skip_signal') and instance.skip_signal:\n                return None\n            return signal_func(sender, instance, *args, **kwargs)\n\n        return _decorator\n\n    return _skip_signal\n\n\ndef download_field_file_to_local_path(field_file: FieldFile, path: str) -> Path:\n    \"\"\"Download entire FieldFile to disk location.\n\n    This overrides `girder_utils.field_file_to_local_path` to download file to\n    local path without a context manager. Cleanup must be handled by caller.\n\n    \"\"\"\n    dest_path = Path(path)\n    dest_path.parent.mkdir(parents=True, exist_ok=True)\n    the_path = Path(dest_path)\n    with field_file.open('rb'):\n        file_obj: File = field_file.file\n        if type(file_obj) is File:\n            # When file_obj is an actual File, (typically backed by FileSystemStorage),\n            # it is already at a stable path on disk.\n            # We must symlink it into the desired path\n            os.symlink(file_obj.name, dest_path)\n            return the_path\n        else:\n            # When file_obj is actually a subclass of File, it only provides a Python\n            # file-like object API. So, it must be copied to a stable path.\n            with open(dest_path, 'wb') as dest_stream:\n                shutil.copyfileobj(file_obj, dest_stream)\n                dest_stream.flush()\n            return the_path\n\n\ndef clean_file_cache(override_target=None):\n    \"\"\"Clean the file cache to achieve RGD_TARGET_AVAILABLE_CACHE (Gb).\n\n    Return\n    ------\n    A tuple of the starting and ending free space in bytes.\n\n    \"\"\"\n    cache = get_cache_dir()\n    # Sort each directory by mtime\n    paths = sorted(Path(cache).iterdir(), key=os.path.getmtime)  # This sorts oldest to latest\n    # While free space is not enough, remove directories until all ar gone\n    initial = psutil.disk_usage(cache).free\n    target = override_target or getattr(settings, 'RGD_TARGET_AVAILABLE_CACHE', 2)\n    logger.info(f'Cleaning file cache... Starting free space is {initial} bytes.')\n    while psutil.disk_usage(cache).free * 1e-9 < target:\n        if not len(paths):\n            # If we delete everything and still cannot acheive target, warn\n            logger.error(\n                f'Target cache free space of {target * 1e9} bytes not achieved when empty.'\n            )\n            break\n        path = paths.pop(0)\n        # Check if resource is locked and skip if so\n        lock = get_file_lock(path)\n        try:\n            with lock.acquire(timeout=0):\n                if os.path.isdir(path):\n                    shutil.rmtree(path)\n                else:\n                    os.remove(path)\n            # Remove the lockfile as well\n            os.remove(lock.lock_file)\n            logger.info(f'removed: {path}')\n        except Timeout:\n            # Another task holds the lock on this file/directory: do not delete\n            logger.info(f'File is locked, skipping: {path}')\n            pass\n    free = psutil.disk_usage(cache).free\n    logger.debug(f'Finished cleaning file cache. Available free space is {free} bytes.')\n    return initial, free\n\n\ndef purge_file_cache():\n    \"\"\"Completely purge all files from the file cache.\n\n    This should be used with extreme caution, it will delete files that could\n    be in use.\n\n    \"\"\"\n    cache = get_cache_dir()\n    shutil.rmtree(cache)\n    cache = get_cache_dir()  # Return the cache dir so that a fresh directory is created.\n    logger.debug(\n        f'Purged file cache. Available free space is {psutil.disk_usage(cache).free} bytes.'\n    )\n    return cache\n", "path": "django-rgd/rgd/utility.py"}]}