We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Remove countdown in build_images 
This task should work synchronously, and not rely on a countdown timer. Maybe create all the `RawImageFile`s first, then add these at the create time of `RawImageUploadSession`. Or, don't rely on a post_save signal, but manually invoke this when all of the database objects are ready.

This will then simplify the test code.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `app/grandchallenge/cases/signals.py`
Content:
```
1 from django.db.models.signals import post_save
2 from django.dispatch import receiver
3 
4 from grandchallenge.cases.models import (
5     RawImageUploadSession, UPLOAD_SESSION_STATE
6 )
7 from grandchallenge.cases.tasks import build_images
8 from grandchallenge.core.utils import disable_for_loaddata
9 
10 
11 @receiver(post_save, sender=RawImageUploadSession)
12 @disable_for_loaddata
13 def queue_build_image_job(
14         instance: RawImageUploadSession=None, created: bool=False,
15         *_, **__):
16     if created:
17         try:
18             task = build_images.apply_async(
19                 args=(instance.pk,),
20                 countdown=5, # Wait a bit - immediate start won't work!
21             )
22             instance.session_state = UPLOAD_SESSION_STATE.queued
23             instance.processing_task = task.id
24             instance.save()
25         except Exception as e:
26             instance.session_state = UPLOAD_SESSION_STATE.stopped
27             instance.error_message = f"Could not start job: {e}"
28             instance.save()
29             raise e
30
```

Path: `app/grandchallenge/cases/forms.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 from typing import List
3 
4 from crispy_forms.helper import FormHelper
5 from crispy_forms.layout import Submit
6 from django import forms
7 from django.db import transaction
8 
9 from grandchallenge.cases.models import RawImageUploadSession, RawImageFile
10 from grandchallenge.jqfileupload.filters import reject_duplicate_filenames
11 from grandchallenge.jqfileupload.widgets import uploader
12 from grandchallenge.jqfileupload.widgets.uploader import (
13     UploadedAjaxFileList, StagedAjaxFile
14 )
15 
16 upload_raw_files_widget = uploader.AjaxUploadWidget(
17     ajax_target_path="ajax/raw_files/",
18     multifile=True,
19     auto_commit=False,
20     upload_validators=[
21         reject_duplicate_filenames,
22     ],
23 )
24 
25 
26 class UploadRawImagesForm(forms.ModelForm):
27     files = UploadedAjaxFileList(
28         widget=upload_raw_files_widget,
29         label="Image files",
30         help_text=(
31             'Upload images for creating a new archive'
32         ),
33     )
34 
35     def __init__(self, *args, **kwargs):
36         super(UploadRawImagesForm, self).__init__(*args, **kwargs)
37         self.helper = FormHelper()
38         self.helper.add_input(Submit("save", "Submit"))
39 
40     def save(self, commit=True):
41         with transaction.atomic():
42             instance = super(UploadRawImagesForm, self).save(commit=commit)
43 
44             # Create links between the created session and all uploaded files
45             uploaded_files = self.cleaned_data[
46                 "files"]  # type: List[StagedAjaxFile]
47 
48             for uploaded_file in uploaded_files:
49                 RawImageFile.objects.create(
50                     upload_session=instance,
51                     filename=uploaded_file.name,
52                     staged_file_id=uploaded_file.uuid,
53                 )
54 
55         return instance
56 
57     class Meta:
58         model = RawImageUploadSession
59         fields = ['files']
60
```

Path: `app/grandchallenge/cases/tasks.py`
Content:
```
1 import shutil
2 from pathlib import Path
3 from tempfile import mkdtemp
4 from typing import Tuple, Sequence
5 from uuid import UUID
6 
7 from celery import shared_task
8 from django.db import transaction
9 
10 from grandchallenge.cases.image_builders import ImageBuilderResult
11 from grandchallenge.cases.image_builders.metaio_mhd_mha import (
12     image_builder_mhd
13 )
14 from grandchallenge.cases.log import logger
15 from grandchallenge.cases.models import (
16     RawImageUploadSession, UPLOAD_SESSION_STATE, Image, ImageFile, RawImageFile
17 )
18 from grandchallenge.jqfileupload.widgets.uploader import (
19     StagedAjaxFile, NotFoundError
20 )
21 
22 
23 class ProvisioningError(Exception): pass
24 
25 
26 def populate_provisioning_directory(
27         raw_files: Sequence[RawImageFile],
28         provisioning_dir: Path):
29     """
30     Provisions provisioning_dir with the files associated using the given
31     list of RawImageFile objects.
32 
33     Parameters
34     ----------
35     raw_files:
36         The list of RawImageFile that should be saved in the target
37         directory.
38 
39     provisioning_dir: Path
40         The path where to copy the files.
41 
42     Raises
43     ------
44     ProvisioningError:
45         Raised when not all files could be copied to the provisioning directory.
46     """
47     provisioning_dir = Path(provisioning_dir)
48 
49     def copy_to_tmpdir(image_file: RawImageFile):
50         staged_file = StagedAjaxFile(image_file.staged_file_id)
51         if not staged_file.exists:
52             raise ValueError(
53                 f"staged file {image_file.staged_file_id} does not exist")
54 
55         with open(provisioning_dir / staged_file.name, "wb") as dest_file:
56             with staged_file.open() as src_file:
57                 BUFFER_SIZE = 0x10000
58                 first = True
59                 while first or (len(buffer) >= BUFFER_SIZE):
60                     first = False
61                     buffer = src_file.read(BUFFER_SIZE)
62                     dest_file.write(buffer)
63 
64     exceptions_raised = 0
65     for raw_file in raw_files:
66         try:
67             copy_to_tmpdir(raw_file)
68         except Exception as e:
69             logger.exception(
70                 f"populate_provisioning_directory exception "
71                 f"for file: '{raw_file.filename}'")
72             exceptions_raised += 1
73 
74     if exceptions_raised > 0:
75         raise ProvisioningError(
76             f"{exceptions_raised} errors occurred during provisioning of the "
77             f"image construction directory")
78 
79 
80 @transaction.atomic
81 def store_image(image: Image, all_image_files: Sequence[ImageFile]):
82     """
83     Stores an image in the database in a single transaction (or fails
84     accordingly). Associated image files are extracted from the
85     all_image_files argument and stored together with the image itself
86     in a single transaction.
87 
88     Parameters
89     ----------
90     image: :class:`Image`
91         The image to store. The actual image files that are stored are extracted
92         from the second argument.
93 
94     all_image_files: list of :class:`ImageFile`
95         An unordered list of ImageFile objects that might or might not belong
96         to the image provided as the first argument. The function automatically
97         extracts related images from the all_image_files argument to store
98         alongside the given image.
99     """
100     associated_files = [
101         _if for _if in all_image_files
102         if _if.image == image
103     ]
104     image.save()
105     for af in associated_files:
106         af.save()
107 
108 
109 IMAGE_BUILDER_ALGORITHMS = [
110     image_builder_mhd
111 ]
112 
113 
114 def remove_duplicate_files(session_files: Sequence[RawImageFile]) -> Tuple[
115     Sequence[RawImageFile], Sequence[RawImageFile]]:
116     """
117     Filters the given sequence of RawImageFile objects and removes all files
118     that have a nun-unqie filename.
119 
120     Parameters
121     ----------
122     session_files: Sequence[RawImageFile]
123         List of RawImageFile objects thats filenames should be checked for
124         uniqueness.
125 
126     Returns
127     -------
128     Two Sequence[RawImageFile]. The first sequence is the filtered session_files
129     list, the second list is a list of duplicates that were removed.
130     """
131     filename_lookup = {}
132     duplicates = []
133     for file in session_files:
134         if file.filename in filename_lookup:
135             duplicates.append(file)
136 
137             looked_up_file = filename_lookup[file.filename]
138             if looked_up_file is not None:
139                 duplicates.append(looked_up_file)
140                 filename_lookup[file.filename] = None
141         else:
142             filename_lookup[file.filename] = file
143     return (
144         tuple(x for x in filename_lookup.values() if x is not None),
145         tuple(duplicates)
146     )
147 
148 
149 @shared_task(
150     autoretry_for=(RawImageUploadSession.DoesNotExist,),
151     default_retry_delay=60,
152     retry_kwargs={
153         'max_retries': 15
154     })
155 def build_images(upload_session_uuid: UUID):
156     """
157     Task which analyzes an upload session and attempts to extract and store
158     detected images assembled from files uploaded in the image session.
159 
160     The task updates the state-filed of the associated
161     :class:`RawImageUploadSession` to indicate if it is running or has finished
162     computing.
163 
164     Results are stored in:
165     - `RawImageUploadSession.error_message` if a general error occurred during
166         processing.
167     - The `RawImageFile.error` field of associated `RawImageFile` objects,
168         in case files could not be processed.
169 
170     The operation of building images will delete associated `StagedAjaxFile`s
171     of analyzed images in order to free up space on the server (only done if the
172     function does not error out).
173 
174     If a job fails due to a RawImageUploadSession.DoesNotExist error, the
175     job is queued for a retry (max 15 times).
176 
177     Parameters
178     ----------
179     upload_session_uuid: UUID
180         The uuid of the upload sessions that should be analyzed.
181     """
182     upload_session = RawImageUploadSession.objects.get(pk=upload_session_uuid)
183     upload_session: RawImageUploadSession
184 
185     if upload_session.session_state == UPLOAD_SESSION_STATE.queued:
186         tmp_dir = Path(mkdtemp(prefix="construct_image_volumes-"))
187         try:
188             try:
189                 upload_session.session_state = UPLOAD_SESSION_STATE.running
190                 upload_session.save()
191 
192                 session_files = RawImageFile.objects.filter(
193                     upload_session=upload_session.pk).all()
194                 session_files: Tuple[RawImageFile]
195 
196                 session_files, duplicates = \
197                     remove_duplicate_files(session_files)
198                 for duplicate in duplicates:
199                     duplicate: RawImageFile
200                     duplicate.error = "Filename not unique"
201                     saf = StagedAjaxFile(duplicate.staged_file_id)
202                     duplicate.staged_file_id = None
203                     saf.delete()
204                     duplicate.save()
205 
206                 populate_provisioning_directory(session_files, tmp_dir)
207 
208                 filename_lookup = {
209                     StagedAjaxFile(
210                         raw_image_file.staged_file_id).name: raw_image_file
211                     for raw_image_file in session_files
212                 }
213                 unconsumed_filenames = set(filename_lookup.keys())
214 
215                 collected_images = []
216                 collected_associated_files = []
217                 for algorithm in IMAGE_BUILDER_ALGORITHMS:
218                     algorithm_result = algorithm(tmp_dir)
219                     algorithm_result: ImageBuilderResult
220 
221                     collected_images += list(algorithm_result.new_images)
222                     collected_associated_files += \
223                         list(algorithm_result.new_image_files)
224 
225                     for filename in algorithm_result.consumed_files:
226                         if filename in unconsumed_filenames:
227                             unconsumed_filenames.remove(filename)
228                     for filename, msg in algorithm_result.file_errors_map.items():
229                         if filename in unconsumed_filenames:
230                             unconsumed_filenames.remove(filename)
231                             raw_image = filename_lookup[filename]
232                             raw_file: RawImageFile
233                             raw_image.error = str(msg)[:256]
234                             raw_image.save()
235 
236                 for image in collected_images:
237                     image.origin = upload_session
238                     store_image(image, collected_associated_files)
239                 for unconsumed_filename in unconsumed_filenames:
240                     raw_file = filename_lookup[unconsumed_filename]
241                     raw_file.error = \
242                         "File could not be processed by any image builder"
243 
244                 # Delete any touched file data
245                 for file in session_files:
246                     try:
247                         saf = StagedAjaxFile(file.staged_file_id)
248                         file.staged_file_id = None
249                         saf.delete()
250                         file.save()
251                     except NotFoundError:
252                         pass
253             except Exception as e:
254                 upload_session.error_message = str(e)
255         finally:
256             if tmp_dir is not None:
257                 shutil.rmtree(tmp_dir)
258 
259             upload_session.session_state = UPLOAD_SESSION_STATE.stopped
260             upload_session.save()
261
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/app/grandchallenge/cases/forms.py b/app/grandchallenge/cases/forms.py
--- a/app/grandchallenge/cases/forms.py
+++ b/app/grandchallenge/cases/forms.py
@@ -33,24 +33,29 @@
     )
 
     def __init__(self, *args, **kwargs):
-        super(UploadRawImagesForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.helper = FormHelper()
         self.helper.add_input(Submit("save", "Submit"))
 
     def save(self, commit=True):
-        with transaction.atomic():
-            instance = super(UploadRawImagesForm, self).save(commit=commit)
-
-            # Create links between the created session and all uploaded files
-            uploaded_files = self.cleaned_data[
-                "files"]  # type: List[StagedAjaxFile]
-
-            for uploaded_file in uploaded_files:
-                RawImageFile.objects.create(
-                    upload_session=instance,
-                    filename=uploaded_file.name,
-                    staged_file_id=uploaded_file.uuid,
-                )
+        instance = super().save(commit=False)
+
+        # Create links between the created session and all uploaded files
+        uploaded_files = self.cleaned_data[
+            "files"]  # type: List[StagedAjaxFile]
+
+        raw_files = [
+            RawImageFile(
+                upload_session=instance,
+                filename=uploaded_file.name,
+                staged_file_id=uploaded_file.uuid,
+            ) for uploaded_file in uploaded_files
+        ]
+
+        if commit:
+            with transaction.atomic():
+                instance.save()
+                RawImageFile.objects.bulk_create(raw_files)
 
         return instance
 
diff --git a/app/grandchallenge/cases/signals.py b/app/grandchallenge/cases/signals.py
--- a/app/grandchallenge/cases/signals.py
+++ b/app/grandchallenge/cases/signals.py
@@ -11,17 +11,20 @@
 @receiver(post_save, sender=RawImageUploadSession)
 @disable_for_loaddata
 def queue_build_image_job(
-        instance: RawImageUploadSession=None, created: bool=False,
-        *_, **__):
+        instance: RawImageUploadSession = None, created: bool = False, *_, **__
+):
     if created:
         try:
-            task = build_images.apply_async(
-                args=(instance.pk,),
-                countdown=5, # Wait a bit - immediate start won't work!
+
+            RawImageUploadSession.objects.filter(pk=instance.pk).update(
+                session_state=UPLOAD_SESSION_STATE.queued,
+                processing_task=instance.pk
             )
-            instance.session_state = UPLOAD_SESSION_STATE.queued
-            instance.processing_task = task.id
-            instance.save()
+
+            build_images.apply_async(
+                task_id=str(instance.pk), args=(instance.pk,),
+            )
+
         except Exception as e:
             instance.session_state = UPLOAD_SESSION_STATE.stopped
             instance.error_message = f"Could not start job: {e}"
diff --git a/app/grandchallenge/cases/tasks.py b/app/grandchallenge/cases/tasks.py
--- a/app/grandchallenge/cases/tasks.py
+++ b/app/grandchallenge/cases/tasks.py
@@ -146,12 +146,7 @@
     )
 
 
-@shared_task(
-    autoretry_for=(RawImageUploadSession.DoesNotExist,),
-    default_retry_delay=60,
-    retry_kwargs={
-        'max_retries': 15
-    })
+@shared_task
 def build_images(upload_session_uuid: UUID):
     """
     Task which analyzes an upload session and attempts to extract and store

{"golden_diff": "diff --git a/app/grandchallenge/cases/forms.py b/app/grandchallenge/cases/forms.py\n--- a/app/grandchallenge/cases/forms.py\n+++ b/app/grandchallenge/cases/forms.py\n@@ -33,24 +33,29 @@\n     )\n \n     def __init__(self, *args, **kwargs):\n-        super(UploadRawImagesForm, self).__init__(*args, **kwargs)\n+        super().__init__(*args, **kwargs)\n         self.helper = FormHelper()\n         self.helper.add_input(Submit(\"save\", \"Submit\"))\n \n     def save(self, commit=True):\n-        with transaction.atomic():\n-            instance = super(UploadRawImagesForm, self).save(commit=commit)\n-\n-            # Create links between the created session and all uploaded files\n-            uploaded_files = self.cleaned_data[\n-                \"files\"]  # type: List[StagedAjaxFile]\n-\n-            for uploaded_file in uploaded_files:\n-                RawImageFile.objects.create(\n-                    upload_session=instance,\n-                    filename=uploaded_file.name,\n-                    staged_file_id=uploaded_file.uuid,\n-                )\n+        instance = super().save(commit=False)\n+\n+        # Create links between the created session and all uploaded files\n+        uploaded_files = self.cleaned_data[\n+            \"files\"]  # type: List[StagedAjaxFile]\n+\n+        raw_files = [\n+            RawImageFile(\n+                upload_session=instance,\n+                filename=uploaded_file.name,\n+                staged_file_id=uploaded_file.uuid,\n+            ) for uploaded_file in uploaded_files\n+        ]\n+\n+        if commit:\n+            with transaction.atomic():\n+                instance.save()\n+                RawImageFile.objects.bulk_create(raw_files)\n \n         return instance\n \ndiff --git a/app/grandchallenge/cases/signals.py b/app/grandchallenge/cases/signals.py\n--- a/app/grandchallenge/cases/signals.py\n+++ b/app/grandchallenge/cases/signals.py\n@@ -11,17 +11,20 @@\n @receiver(post_save, sender=RawImageUploadSession)\n @disable_for_loaddata\n def queue_build_image_job(\n-        instance: RawImageUploadSession=None, created: bool=False,\n-        *_, **__):\n+        instance: RawImageUploadSession = None, created: bool = False, *_, **__\n+):\n     if created:\n         try:\n-            task = build_images.apply_async(\n-                args=(instance.pk,),\n-                countdown=5, # Wait a bit - immediate start won't work!\n+\n+            RawImageUploadSession.objects.filter(pk=instance.pk).update(\n+                session_state=UPLOAD_SESSION_STATE.queued,\n+                processing_task=instance.pk\n             )\n-            instance.session_state = UPLOAD_SESSION_STATE.queued\n-            instance.processing_task = task.id\n-            instance.save()\n+\n+            build_images.apply_async(\n+                task_id=str(instance.pk), args=(instance.pk,),\n+            )\n+\n         except Exception as e:\n             instance.session_state = UPLOAD_SESSION_STATE.stopped\n             instance.error_message = f\"Could not start job: {e}\"\ndiff --git a/app/grandchallenge/cases/tasks.py b/app/grandchallenge/cases/tasks.py\n--- a/app/grandchallenge/cases/tasks.py\n+++ b/app/grandchallenge/cases/tasks.py\n@@ -146,12 +146,7 @@\n     )\n \n \n-@shared_task(\n-    autoretry_for=(RawImageUploadSession.DoesNotExist,),\n-    default_retry_delay=60,\n-    retry_kwargs={\n-        'max_retries': 15\n-    })\n+@shared_task\n def build_images(upload_session_uuid: UUID):\n     \"\"\"\n     Task which analyzes an upload session and attempts to extract and store\n", "issue": "Remove countdown in build_images \nThis task should work synchronously, and not rely on a countdown timer. Maybe create all the `RawImageFile`s first, then add these at the create time of `RawImageUploadSession`. Or, don't rely on a post_save signal, but manually invoke this when all of the database objects are ready.\r\n\r\nThis will then simplify the test code.\n", "before_files": [{"content": "from django.db.models.signals import post_save\nfrom django.dispatch import receiver\n\nfrom grandchallenge.cases.models import (\n    RawImageUploadSession, UPLOAD_SESSION_STATE\n)\nfrom grandchallenge.cases.tasks import build_images\nfrom grandchallenge.core.utils import disable_for_loaddata\n\n\n@receiver(post_save, sender=RawImageUploadSession)\n@disable_for_loaddata\ndef queue_build_image_job(\n        instance: RawImageUploadSession=None, created: bool=False,\n        *_, **__):\n    if created:\n        try:\n            task = build_images.apply_async(\n                args=(instance.pk,),\n                countdown=5, # Wait a bit - immediate start won't work!\n            )\n            instance.session_state = UPLOAD_SESSION_STATE.queued\n            instance.processing_task = task.id\n            instance.save()\n        except Exception as e:\n            instance.session_state = UPLOAD_SESSION_STATE.stopped\n            instance.error_message = f\"Could not start job: {e}\"\n            instance.save()\n            raise e\n", "path": "app/grandchallenge/cases/signals.py"}, {"content": "# -*- coding: utf-8 -*-\nfrom typing import List\n\nfrom crispy_forms.helper import FormHelper\nfrom crispy_forms.layout import Submit\nfrom django import forms\nfrom django.db import transaction\n\nfrom grandchallenge.cases.models import RawImageUploadSession, RawImageFile\nfrom grandchallenge.jqfileupload.filters import reject_duplicate_filenames\nfrom grandchallenge.jqfileupload.widgets import uploader\nfrom grandchallenge.jqfileupload.widgets.uploader import (\n    UploadedAjaxFileList, StagedAjaxFile\n)\n\nupload_raw_files_widget = uploader.AjaxUploadWidget(\n    ajax_target_path=\"ajax/raw_files/\",\n    multifile=True,\n    auto_commit=False,\n    upload_validators=[\n        reject_duplicate_filenames,\n    ],\n)\n\n\nclass UploadRawImagesForm(forms.ModelForm):\n    files = UploadedAjaxFileList(\n        widget=upload_raw_files_widget,\n        label=\"Image files\",\n        help_text=(\n            'Upload images for creating a new archive'\n        ),\n    )\n\n    def __init__(self, *args, **kwargs):\n        super(UploadRawImagesForm, self).__init__(*args, **kwargs)\n        self.helper = FormHelper()\n        self.helper.add_input(Submit(\"save\", \"Submit\"))\n\n    def save(self, commit=True):\n        with transaction.atomic():\n            instance = super(UploadRawImagesForm, self).save(commit=commit)\n\n            # Create links between the created session and all uploaded files\n            uploaded_files = self.cleaned_data[\n                \"files\"]  # type: List[StagedAjaxFile]\n\n            for uploaded_file in uploaded_files:\n                RawImageFile.objects.create(\n                    upload_session=instance,\n                    filename=uploaded_file.name,\n                    staged_file_id=uploaded_file.uuid,\n                )\n\n        return instance\n\n    class Meta:\n        model = RawImageUploadSession\n        fields = ['files']\n", "path": "app/grandchallenge/cases/forms.py"}, {"content": "import shutil\nfrom pathlib import Path\nfrom tempfile import mkdtemp\nfrom typing import Tuple, Sequence\nfrom uuid import UUID\n\nfrom celery import shared_task\nfrom django.db import transaction\n\nfrom grandchallenge.cases.image_builders import ImageBuilderResult\nfrom grandchallenge.cases.image_builders.metaio_mhd_mha import (\n    image_builder_mhd\n)\nfrom grandchallenge.cases.log import logger\nfrom grandchallenge.cases.models import (\n    RawImageUploadSession, UPLOAD_SESSION_STATE, Image, ImageFile, RawImageFile\n)\nfrom grandchallenge.jqfileupload.widgets.uploader import (\n    StagedAjaxFile, NotFoundError\n)\n\n\nclass ProvisioningError(Exception): pass\n\n\ndef populate_provisioning_directory(\n        raw_files: Sequence[RawImageFile],\n        provisioning_dir: Path):\n    \"\"\"\n    Provisions provisioning_dir with the files associated using the given\n    list of RawImageFile objects.\n\n    Parameters\n    ----------\n    raw_files:\n        The list of RawImageFile that should be saved in the target\n        directory.\n\n    provisioning_dir: Path\n        The path where to copy the files.\n\n    Raises\n    ------\n    ProvisioningError:\n        Raised when not all files could be copied to the provisioning directory.\n    \"\"\"\n    provisioning_dir = Path(provisioning_dir)\n\n    def copy_to_tmpdir(image_file: RawImageFile):\n        staged_file = StagedAjaxFile(image_file.staged_file_id)\n        if not staged_file.exists:\n            raise ValueError(\n                f\"staged file {image_file.staged_file_id} does not exist\")\n\n        with open(provisioning_dir / staged_file.name, \"wb\") as dest_file:\n            with staged_file.open() as src_file:\n                BUFFER_SIZE = 0x10000\n                first = True\n                while first or (len(buffer) >= BUFFER_SIZE):\n                    first = False\n                    buffer = src_file.read(BUFFER_SIZE)\n                    dest_file.write(buffer)\n\n    exceptions_raised = 0\n    for raw_file in raw_files:\n        try:\n            copy_to_tmpdir(raw_file)\n        except Exception as e:\n            logger.exception(\n                f\"populate_provisioning_directory exception \"\n                f\"for file: '{raw_file.filename}'\")\n            exceptions_raised += 1\n\n    if exceptions_raised > 0:\n        raise ProvisioningError(\n            f\"{exceptions_raised} errors occurred during provisioning of the \"\n            f\"image construction directory\")\n\n\n@transaction.atomic\ndef store_image(image: Image, all_image_files: Sequence[ImageFile]):\n    \"\"\"\n    Stores an image in the database in a single transaction (or fails\n    accordingly). Associated image files are extracted from the\n    all_image_files argument and stored together with the image itself\n    in a single transaction.\n\n    Parameters\n    ----------\n    image: :class:`Image`\n        The image to store. The actual image files that are stored are extracted\n        from the second argument.\n\n    all_image_files: list of :class:`ImageFile`\n        An unordered list of ImageFile objects that might or might not belong\n        to the image provided as the first argument. The function automatically\n        extracts related images from the all_image_files argument to store\n        alongside the given image.\n    \"\"\"\n    associated_files = [\n        _if for _if in all_image_files\n        if _if.image == image\n    ]\n    image.save()\n    for af in associated_files:\n        af.save()\n\n\nIMAGE_BUILDER_ALGORITHMS = [\n    image_builder_mhd\n]\n\n\ndef remove_duplicate_files(session_files: Sequence[RawImageFile]) -> Tuple[\n    Sequence[RawImageFile], Sequence[RawImageFile]]:\n    \"\"\"\n    Filters the given sequence of RawImageFile objects and removes all files\n    that have a nun-unqie filename.\n\n    Parameters\n    ----------\n    session_files: Sequence[RawImageFile]\n        List of RawImageFile objects thats filenames should be checked for\n        uniqueness.\n\n    Returns\n    -------\n    Two Sequence[RawImageFile]. The first sequence is the filtered session_files\n    list, the second list is a list of duplicates that were removed.\n    \"\"\"\n    filename_lookup = {}\n    duplicates = []\n    for file in session_files:\n        if file.filename in filename_lookup:\n            duplicates.append(file)\n\n            looked_up_file = filename_lookup[file.filename]\n            if looked_up_file is not None:\n                duplicates.append(looked_up_file)\n                filename_lookup[file.filename] = None\n        else:\n            filename_lookup[file.filename] = file\n    return (\n        tuple(x for x in filename_lookup.values() if x is not None),\n        tuple(duplicates)\n    )\n\n\n@shared_task(\n    autoretry_for=(RawImageUploadSession.DoesNotExist,),\n    default_retry_delay=60,\n    retry_kwargs={\n        'max_retries': 15\n    })\ndef build_images(upload_session_uuid: UUID):\n    \"\"\"\n    Task which analyzes an upload session and attempts to extract and store\n    detected images assembled from files uploaded in the image session.\n\n    The task updates the state-filed of the associated\n    :class:`RawImageUploadSession` to indicate if it is running or has finished\n    computing.\n\n    Results are stored in:\n    - `RawImageUploadSession.error_message` if a general error occurred during\n        processing.\n    - The `RawImageFile.error` field of associated `RawImageFile` objects,\n        in case files could not be processed.\n\n    The operation of building images will delete associated `StagedAjaxFile`s\n    of analyzed images in order to free up space on the server (only done if the\n    function does not error out).\n\n    If a job fails due to a RawImageUploadSession.DoesNotExist error, the\n    job is queued for a retry (max 15 times).\n\n    Parameters\n    ----------\n    upload_session_uuid: UUID\n        The uuid of the upload sessions that should be analyzed.\n    \"\"\"\n    upload_session = RawImageUploadSession.objects.get(pk=upload_session_uuid)\n    upload_session: RawImageUploadSession\n\n    if upload_session.session_state == UPLOAD_SESSION_STATE.queued:\n        tmp_dir = Path(mkdtemp(prefix=\"construct_image_volumes-\"))\n        try:\n            try:\n                upload_session.session_state = UPLOAD_SESSION_STATE.running\n                upload_session.save()\n\n                session_files = RawImageFile.objects.filter(\n                    upload_session=upload_session.pk).all()\n                session_files: Tuple[RawImageFile]\n\n                session_files, duplicates = \\\n                    remove_duplicate_files(session_files)\n                for duplicate in duplicates:\n                    duplicate: RawImageFile\n                    duplicate.error = \"Filename not unique\"\n                    saf = StagedAjaxFile(duplicate.staged_file_id)\n                    duplicate.staged_file_id = None\n                    saf.delete()\n                    duplicate.save()\n\n                populate_provisioning_directory(session_files, tmp_dir)\n\n                filename_lookup = {\n                    StagedAjaxFile(\n                        raw_image_file.staged_file_id).name: raw_image_file\n                    for raw_image_file in session_files\n                }\n                unconsumed_filenames = set(filename_lookup.keys())\n\n                collected_images = []\n                collected_associated_files = []\n                for algorithm in IMAGE_BUILDER_ALGORITHMS:\n                    algorithm_result = algorithm(tmp_dir)\n                    algorithm_result: ImageBuilderResult\n\n                    collected_images += list(algorithm_result.new_images)\n                    collected_associated_files += \\\n                        list(algorithm_result.new_image_files)\n\n                    for filename in algorithm_result.consumed_files:\n                        if filename in unconsumed_filenames:\n                            unconsumed_filenames.remove(filename)\n                    for filename, msg in algorithm_result.file_errors_map.items():\n                        if filename in unconsumed_filenames:\n                            unconsumed_filenames.remove(filename)\n                            raw_image = filename_lookup[filename]\n                            raw_file: RawImageFile\n                            raw_image.error = str(msg)[:256]\n                            raw_image.save()\n\n                for image in collected_images:\n                    image.origin = upload_session\n                    store_image(image, collected_associated_files)\n                for unconsumed_filename in unconsumed_filenames:\n                    raw_file = filename_lookup[unconsumed_filename]\n                    raw_file.error = \\\n                        \"File could not be processed by any image builder\"\n\n                # Delete any touched file data\n                for file in session_files:\n                    try:\n                        saf = StagedAjaxFile(file.staged_file_id)\n                        file.staged_file_id = None\n                        saf.delete()\n                        file.save()\n                    except NotFoundError:\n                        pass\n            except Exception as e:\n                upload_session.error_message = str(e)\n        finally:\n            if tmp_dir is not None:\n                shutil.rmtree(tmp_dir)\n\n            upload_session.session_state = UPLOAD_SESSION_STATE.stopped\n            upload_session.save()\n", "path": "app/grandchallenge/cases/tasks.py"}], "after_files": [{"content": "from django.db.models.signals import post_save\nfrom django.dispatch import receiver\n\nfrom grandchallenge.cases.models import (\n    RawImageUploadSession, UPLOAD_SESSION_STATE\n)\nfrom grandchallenge.cases.tasks import build_images\nfrom grandchallenge.core.utils import disable_for_loaddata\n\n\n@receiver(post_save, sender=RawImageUploadSession)\n@disable_for_loaddata\ndef queue_build_image_job(\n        instance: RawImageUploadSession = None, created: bool = False, *_, **__\n):\n    if created:\n        try:\n\n            RawImageUploadSession.objects.filter(pk=instance.pk).update(\n                session_state=UPLOAD_SESSION_STATE.queued,\n                processing_task=instance.pk\n            )\n\n            build_images.apply_async(\n                task_id=str(instance.pk), args=(instance.pk,),\n            )\n\n        except Exception as e:\n            instance.session_state = UPLOAD_SESSION_STATE.stopped\n            instance.error_message = f\"Could not start job: {e}\"\n            instance.save()\n            raise e\n", "path": "app/grandchallenge/cases/signals.py"}, {"content": "# -*- coding: utf-8 -*-\nfrom typing import List\n\nfrom crispy_forms.helper import FormHelper\nfrom crispy_forms.layout import Submit\nfrom django import forms\nfrom django.db import transaction\n\nfrom grandchallenge.cases.models import RawImageUploadSession, RawImageFile\nfrom grandchallenge.jqfileupload.filters import reject_duplicate_filenames\nfrom grandchallenge.jqfileupload.widgets import uploader\nfrom grandchallenge.jqfileupload.widgets.uploader import (\n    UploadedAjaxFileList, StagedAjaxFile\n)\n\nupload_raw_files_widget = uploader.AjaxUploadWidget(\n    ajax_target_path=\"ajax/raw_files/\",\n    multifile=True,\n    auto_commit=False,\n    upload_validators=[\n        reject_duplicate_filenames,\n    ],\n)\n\n\nclass UploadRawImagesForm(forms.ModelForm):\n    files = UploadedAjaxFileList(\n        widget=upload_raw_files_widget,\n        label=\"Image files\",\n        help_text=(\n            'Upload images for creating a new archive'\n        ),\n    )\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.helper = FormHelper()\n        self.helper.add_input(Submit(\"save\", \"Submit\"))\n\n    def save(self, commit=True):\n        instance = super().save(commit=False)\n\n        # Create links between the created session and all uploaded files\n        uploaded_files = self.cleaned_data[\n            \"files\"]  # type: List[StagedAjaxFile]\n\n        raw_files = [\n            RawImageFile(\n                upload_session=instance,\n                filename=uploaded_file.name,\n                staged_file_id=uploaded_file.uuid,\n            ) for uploaded_file in uploaded_files\n        ]\n\n        if commit:\n            with transaction.atomic():\n                instance.save()\n                RawImageFile.objects.bulk_create(raw_files)\n\n        return instance\n\n    class Meta:\n        model = RawImageUploadSession\n        fields = ['files']\n", "path": "app/grandchallenge/cases/forms.py"}, {"content": "import shutil\nfrom pathlib import Path\nfrom tempfile import mkdtemp\nfrom typing import Tuple, Sequence\nfrom uuid import UUID\n\nfrom celery import shared_task\nfrom django.db import transaction\n\nfrom grandchallenge.cases.image_builders import ImageBuilderResult\nfrom grandchallenge.cases.image_builders.metaio_mhd_mha import (\n    image_builder_mhd\n)\nfrom grandchallenge.cases.log import logger\nfrom grandchallenge.cases.models import (\n    RawImageUploadSession, UPLOAD_SESSION_STATE, Image, ImageFile, RawImageFile\n)\nfrom grandchallenge.jqfileupload.widgets.uploader import (\n    StagedAjaxFile, NotFoundError\n)\n\n\nclass ProvisioningError(Exception): pass\n\n\ndef populate_provisioning_directory(\n        raw_files: Sequence[RawImageFile],\n        provisioning_dir: Path):\n    \"\"\"\n    Provisions provisioning_dir with the files associated using the given\n    list of RawImageFile objects.\n\n    Parameters\n    ----------\n    raw_files:\n        The list of RawImageFile that should be saved in the target\n        directory.\n\n    provisioning_dir: Path\n        The path where to copy the files.\n\n    Raises\n    ------\n    ProvisioningError:\n        Raised when not all files could be copied to the provisioning directory.\n    \"\"\"\n    provisioning_dir = Path(provisioning_dir)\n\n    def copy_to_tmpdir(image_file: RawImageFile):\n        staged_file = StagedAjaxFile(image_file.staged_file_id)\n        if not staged_file.exists:\n            raise ValueError(\n                f\"staged file {image_file.staged_file_id} does not exist\")\n\n        with open(provisioning_dir / staged_file.name, \"wb\") as dest_file:\n            with staged_file.open() as src_file:\n                BUFFER_SIZE = 0x10000\n                first = True\n                while first or (len(buffer) >= BUFFER_SIZE):\n                    first = False\n                    buffer = src_file.read(BUFFER_SIZE)\n                    dest_file.write(buffer)\n\n    exceptions_raised = 0\n    for raw_file in raw_files:\n        try:\n            copy_to_tmpdir(raw_file)\n        except Exception as e:\n            logger.exception(\n                f\"populate_provisioning_directory exception \"\n                f\"for file: '{raw_file.filename}'\")\n            exceptions_raised += 1\n\n    if exceptions_raised > 0:\n        raise ProvisioningError(\n            f\"{exceptions_raised} errors occurred during provisioning of the \"\n            f\"image construction directory\")\n\n\n@transaction.atomic\ndef store_image(image: Image, all_image_files: Sequence[ImageFile]):\n    \"\"\"\n    Stores an image in the database in a single transaction (or fails\n    accordingly). Associated image files are extracted from the\n    all_image_files argument and stored together with the image itself\n    in a single transaction.\n\n    Parameters\n    ----------\n    image: :class:`Image`\n        The image to store. The actual image files that are stored are extracted\n        from the second argument.\n\n    all_image_files: list of :class:`ImageFile`\n        An unordered list of ImageFile objects that might or might not belong\n        to the image provided as the first argument. The function automatically\n        extracts related images from the all_image_files argument to store\n        alongside the given image.\n    \"\"\"\n    associated_files = [\n        _if for _if in all_image_files\n        if _if.image == image\n    ]\n    image.save()\n    for af in associated_files:\n        af.save()\n\n\nIMAGE_BUILDER_ALGORITHMS = [\n    image_builder_mhd\n]\n\n\ndef remove_duplicate_files(session_files: Sequence[RawImageFile]) -> Tuple[\n    Sequence[RawImageFile], Sequence[RawImageFile]]:\n    \"\"\"\n    Filters the given sequence of RawImageFile objects and removes all files\n    that have a nun-unqie filename.\n\n    Parameters\n    ----------\n    session_files: Sequence[RawImageFile]\n        List of RawImageFile objects thats filenames should be checked for\n        uniqueness.\n\n    Returns\n    -------\n    Two Sequence[RawImageFile]. The first sequence is the filtered session_files\n    list, the second list is a list of duplicates that were removed.\n    \"\"\"\n    filename_lookup = {}\n    duplicates = []\n    for file in session_files:\n        if file.filename in filename_lookup:\n            duplicates.append(file)\n\n            looked_up_file = filename_lookup[file.filename]\n            if looked_up_file is not None:\n                duplicates.append(looked_up_file)\n                filename_lookup[file.filename] = None\n        else:\n            filename_lookup[file.filename] = file\n    return (\n        tuple(x for x in filename_lookup.values() if x is not None),\n        tuple(duplicates)\n    )\n\n\n@shared_task\ndef build_images(upload_session_uuid: UUID):\n    \"\"\"\n    Task which analyzes an upload session and attempts to extract and store\n    detected images assembled from files uploaded in the image session.\n\n    The task updates the state-filed of the associated\n    :class:`RawImageUploadSession` to indicate if it is running or has finished\n    computing.\n\n    Results are stored in:\n    - `RawImageUploadSession.error_message` if a general error occurred during\n        processing.\n    - The `RawImageFile.error` field of associated `RawImageFile` objects,\n        in case files could not be processed.\n\n    The operation of building images will delete associated `StagedAjaxFile`s\n    of analyzed images in order to free up space on the server (only done if the\n    function does not error out).\n\n    If a job fails due to a RawImageUploadSession.DoesNotExist error, the\n    job is queued for a retry (max 15 times).\n\n    Parameters\n    ----------\n    upload_session_uuid: UUID\n        The uuid of the upload sessions that should be analyzed.\n    \"\"\"\n    upload_session = RawImageUploadSession.objects.get(pk=upload_session_uuid)\n    upload_session: RawImageUploadSession\n\n    if upload_session.session_state == UPLOAD_SESSION_STATE.queued:\n        tmp_dir = Path(mkdtemp(prefix=\"construct_image_volumes-\"))\n        try:\n            try:\n                upload_session.session_state = UPLOAD_SESSION_STATE.running\n                upload_session.save()\n\n                session_files = RawImageFile.objects.filter(\n                    upload_session=upload_session.pk).all()\n                session_files: Tuple[RawImageFile]\n\n                session_files, duplicates = \\\n                    remove_duplicate_files(session_files)\n                for duplicate in duplicates:\n                    duplicate: RawImageFile\n                    duplicate.error = \"Filename not unique\"\n                    saf = StagedAjaxFile(duplicate.staged_file_id)\n                    duplicate.staged_file_id = None\n                    saf.delete()\n                    duplicate.save()\n\n                populate_provisioning_directory(session_files, tmp_dir)\n\n                filename_lookup = {\n                    StagedAjaxFile(\n                        raw_image_file.staged_file_id).name: raw_image_file\n                    for raw_image_file in session_files\n                }\n                unconsumed_filenames = set(filename_lookup.keys())\n\n                collected_images = []\n                collected_associated_files = []\n                for algorithm in IMAGE_BUILDER_ALGORITHMS:\n                    algorithm_result = algorithm(tmp_dir)\n                    algorithm_result: ImageBuilderResult\n\n                    collected_images += list(algorithm_result.new_images)\n                    collected_associated_files += \\\n                        list(algorithm_result.new_image_files)\n\n                    for filename in algorithm_result.consumed_files:\n                        if filename in unconsumed_filenames:\n                            unconsumed_filenames.remove(filename)\n                    for filename, msg in algorithm_result.file_errors_map.items():\n                        if filename in unconsumed_filenames:\n                            unconsumed_filenames.remove(filename)\n                            raw_image = filename_lookup[filename]\n                            raw_file: RawImageFile\n                            raw_image.error = str(msg)[:256]\n                            raw_image.save()\n\n                for image in collected_images:\n                    image.origin = upload_session\n                    store_image(image, collected_associated_files)\n                for unconsumed_filename in unconsumed_filenames:\n                    raw_file = filename_lookup[unconsumed_filename]\n                    raw_file.error = \\\n                        \"File could not be processed by any image builder\"\n\n                # Delete any touched file data\n                for file in session_files:\n                    try:\n                        saf = StagedAjaxFile(file.staged_file_id)\n                        file.staged_file_id = None\n                        saf.delete()\n                        file.save()\n                    except NotFoundError:\n                        pass\n            except Exception as e:\n                upload_session.error_message = str(e)\n        finally:\n            if tmp_dir is not None:\n                shutil.rmtree(tmp_dir)\n\n            upload_session.session_state = UPLOAD_SESSION_STATE.stopped\n            upload_session.save()\n", "path": "app/grandchallenge/cases/tasks.py"}]}