We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Include one-page text per item in BagIt export
**Is your feature request related to a problem? Please describe.**
In the BagIt export which feeds transcriptions into loc.gov, we want to provide one plain text file per item which has all the transcription text from all pages.

**Describe the solution you'd like**
The text file should contain all pages in the item and be named with the item ID. There should be a single attribution at the bottom of all the text.



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `exporter/views.py`
Content:
```
1 import os
2 import re
3 import shutil
4 import tempfile
5 from logging import getLogger
6 
7 import bagit
8 import boto3
9 from django.conf import settings
10 from django.contrib.admin.views.decorators import staff_member_required
11 from django.db.models import OuterRef, Subquery
12 from django.http import HttpResponse, HttpResponseRedirect
13 from django.utils.decorators import method_decorator
14 from django.views.generic import TemplateView
15 from tabular_export.core import export_to_csv_response, flatten_queryset
16 
17 from concordia.models import Asset, Transcription, TranscriptionStatus
18 
19 logger = getLogger(__name__)
20 
21 
22 def get_latest_transcription_data(asset_qs):
23     latest_trans_subquery = (
24         Transcription.objects.filter(asset=OuterRef("pk"))
25         .order_by("-pk")
26         .values("text")
27     )
28 
29     assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))
30     return assets
31 
32 
33 def get_original_asset_id(download_url):
34     """
35     Extract the bit from the download url
36     that identifies this image uniquely on loc.gov
37     """
38     if download_url.startswith("http://tile.loc.gov/"):
39         pattern = r"/service:([A-Za-z0-9:\-]+)/"
40         asset_id = re.search(pattern, download_url)
41         if not asset_id:
42             logger.error(
43                 "Couldn't find a matching asset ID in download URL %s", download_url
44             )
45             raise AssertionError
46         else:
47             matching_asset_id = asset_id.group(1)
48             logger.debug(
49                 "Found asset ID %s in download URL %s", matching_asset_id, download_url
50             )
51             return matching_asset_id
52     else:
53         logger.warning("Download URL doesn't start with tile.loc.gov: %s", download_url)
54         return download_url
55 
56 
57 def do_bagit_export(assets, export_base_dir, export_filename_base):
58     """
59     Executes bagit.py to turn temp directory into LC-specific bagit strucutre.
60     Builds and exports bagit structure as zip.
61     Uploads zip to S3 if configured.
62     """
63 
64     for asset in assets:
65         asset_id = get_original_asset_id(asset.download_url)
66         logger.debug("Exporting asset %s into %s", asset_id, export_base_dir)
67 
68         asset_id = asset_id.replace(":", "/")
69         asset_path, asset_filename = os.path.split(asset_id)
70 
71         dest_path = os.path.join(export_base_dir, asset_path)
72         os.makedirs(dest_path, exist_ok=True)
73 
74         # Build transcription output text file
75         text_output_path = os.path.join(dest_path, "%s.txt" % asset_filename)
76         with open(text_output_path, "w") as f:
77             f.write(asset.latest_transcription or "")
78             if hasattr(settings, "ATTRIBUTION_TEXT"):
79                 f.write("\n\n")
80                 f.write(settings.ATTRIBUTION_TEXT)
81 
82     # Turn Structure into bagit format
83     bagit.make_bag(
84         export_base_dir,
85         {
86             "Content-Access": "web",
87             "Content-Custodian": "dcms",
88             "Content-Process": "crowdsourced",
89             "Content-Type": "textual",
90             "LC-Bag-Id": export_filename_base,
91             "LC-Items": "%d transcriptions" % len(assets),
92             "LC-Project": "gdccrowd",
93             "License-Information": "Public domain",
94         },
95     )
96 
97     # Build .zip file of bagit formatted Campaign Folder
98     archive_name = export_base_dir
99     shutil.make_archive(archive_name, "zip", export_base_dir)
100 
101     export_filename = "%s.zip" % export_filename_base
102 
103     # Upload zip to S3 bucket
104     s3_bucket = getattr(settings, "EXPORT_S3_BUCKET_NAME", None)
105 
106     if s3_bucket:
107         logger.debug("Uploading exported bag to S3 bucket %s", s3_bucket)
108         s3 = boto3.resource("s3")
109         s3.Bucket(s3_bucket).upload_file(
110             "%s.zip" % export_base_dir, "%s" % export_filename
111         )
112 
113         return HttpResponseRedirect(
114             "https://%s.s3.amazonaws.com/%s" % (s3_bucket, export_filename)
115         )
116     else:
117         # Download zip from local storage
118         with open("%s.zip" % export_base_dir, "rb") as zip_file:
119             response = HttpResponse(zip_file, content_type="application/zip")
120         response["Content-Disposition"] = "attachment; filename=%s" % export_filename
121         return response
122 
123 
124 class ExportCampaignToCSV(TemplateView):
125     """
126     Exports the most recent transcription for each asset in a campaign
127     """
128 
129     @method_decorator(staff_member_required)
130     def get(self, request, *args, **kwargs):
131         asset_qs = Asset.objects.filter(
132             item__project__campaign__slug=self.kwargs["campaign_slug"]
133         )
134         assets = get_latest_transcription_data(asset_qs)
135 
136         headers, data = flatten_queryset(
137             assets,
138             field_names=[
139                 "item__project__campaign__title",
140                 "item__project__title",
141                 "item__title",
142                 "item__item_id",
143                 "title",
144                 "transcription_status",
145                 "download_url",
146                 "latest_transcription",
147             ],
148             extra_verbose_names={
149                 "item__project__campaign__title": "Campaign",
150                 "item__project__title": "Project",
151                 "item__title": "Item",
152                 "item__item_id": "ItemId",
153                 "item_id": "ItemId",
154                 "title": "Asset",
155                 "transcription_status": "AssetStatus",
156                 "download_url": "DownloadUrl",
157                 "latest_transcription": "Transcription",
158             },
159         )
160 
161         return export_to_csv_response(
162             "%s.csv" % self.kwargs["campaign_slug"], headers, data
163         )
164 
165 
166 class ExportItemToBagIt(TemplateView):
167     @method_decorator(staff_member_required)
168     def get(self, request, *args, **kwargs):
169         campaign_slug = self.kwargs["campaign_slug"]
170         project_slug = self.kwargs["project_slug"]
171         item_id = self.kwargs["item_id"]
172 
173         asset_qs = Asset.objects.filter(
174             item__project__campaign__slug=campaign_slug,
175             item__project__slug=project_slug,
176             item__item_id=item_id,
177             transcription_status=TranscriptionStatus.COMPLETED,
178         )
179 
180         assets = get_latest_transcription_data(asset_qs)
181 
182         export_filename_base = "%s-%s-%s" % (campaign_slug, project_slug, item_id)
183 
184         with tempfile.TemporaryDirectory(
185             prefix=export_filename_base
186         ) as export_base_dir:
187             return do_bagit_export(assets, export_base_dir, export_filename_base)
188 
189 
190 class ExportProjectToBagIt(TemplateView):
191     @method_decorator(staff_member_required)
192     def get(self, request, *args, **kwargs):
193         campaign_slug = self.kwargs["campaign_slug"]
194         project_slug = self.kwargs["project_slug"]
195         asset_qs = Asset.objects.filter(
196             item__project__campaign__slug=campaign_slug,
197             item__project__slug=project_slug,
198             transcription_status=TranscriptionStatus.COMPLETED,
199         )
200 
201         assets = get_latest_transcription_data(asset_qs)
202 
203         export_filename_base = "%s-%s" % (campaign_slug, project_slug)
204 
205         with tempfile.TemporaryDirectory(
206             prefix=export_filename_base
207         ) as export_base_dir:
208             return do_bagit_export(assets, export_base_dir, export_filename_base)
209 
210 
211 class ExportCampaignToBagit(TemplateView):
212     @method_decorator(staff_member_required)
213     def get(self, request, *args, **kwargs):
214         campaign_slug = self.kwargs["campaign_slug"]
215         asset_qs = Asset.objects.filter(
216             item__project__campaign__slug=campaign_slug,
217             transcription_status=TranscriptionStatus.COMPLETED,
218         )
219 
220         assets = get_latest_transcription_data(asset_qs)
221 
222         export_filename_base = "%s" % (campaign_slug,)
223 
224         with tempfile.TemporaryDirectory(
225             prefix=export_filename_base
226         ) as export_base_dir:
227             return do_bagit_export(assets, export_base_dir, export_filename_base)
228
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/exporter/views.py b/exporter/views.py
--- a/exporter/views.py
+++ b/exporter/views.py
@@ -61,23 +61,40 @@
     Uploads zip to S3 if configured.
     """
 
+    # These assets should already be in the correct order - by item, seequence
     for asset in assets:
         asset_id = get_original_asset_id(asset.download_url)
         logger.debug("Exporting asset %s into %s", asset_id, export_base_dir)
 
         asset_id = asset_id.replace(":", "/")
         asset_path, asset_filename = os.path.split(asset_id)
+        item_path, item_filename = os.path.split(asset_path)
 
-        dest_path = os.path.join(export_base_dir, asset_path)
-        os.makedirs(dest_path, exist_ok=True)
+        asset_dest_path = os.path.join(export_base_dir, asset_path)
+        os.makedirs(asset_dest_path, exist_ok=True)
 
-        # Build transcription output text file
-        text_output_path = os.path.join(dest_path, "%s.txt" % asset_filename)
-        with open(text_output_path, "w") as f:
+        # Build a transcription output text file for each asset
+        asset_text_output_path = os.path.join(
+            asset_dest_path, "%s.txt" % asset_filename
+        )
+        # Write the asset level transcription file
+        with open(asset_text_output_path, "w") as f:
+            f.write(asset.latest_transcription or "")
+
+        # Append this asset transcription to the item transcription
+        item_text_output_path = os.path.join(asset_dest_path, "%s.txt" % item_filename)
+        with open(item_text_output_path, "a") as f:
             f.write(asset.latest_transcription or "")
-            if hasattr(settings, "ATTRIBUTION_TEXT"):
-                f.write("\n\n")
-                f.write(settings.ATTRIBUTION_TEXT)
+            f.write("\n\n")
+
+    # Add attributions to the end of all text files found under asset_dest_path
+    if hasattr(settings, "ATTRIBUTION_TEXT"):
+        for dirpath, dirnames, filenames in os.walk(export_base_dir, topdown=False):
+            for each_text_file in (i for i in filenames if i.endswith(".txt")):
+                this_text_file = os.path.join(dirpath, each_text_file)
+                with open(this_text_file, "a") as f:
+                    f.write("\n\n")
+                    f.write(settings.ATTRIBUTION_TEXT)
 
     # Turn Structure into bagit format
     bagit.make_bag(
@@ -175,7 +192,7 @@
             item__project__slug=project_slug,
             item__item_id=item_id,
             transcription_status=TranscriptionStatus.COMPLETED,
-        )
+        ).order_by("sequence")
 
         assets = get_latest_transcription_data(asset_qs)
 
@@ -196,7 +213,7 @@
             item__project__campaign__slug=campaign_slug,
             item__project__slug=project_slug,
             transcription_status=TranscriptionStatus.COMPLETED,
-        )
+        ).order_by("item", "sequence")
 
         assets = get_latest_transcription_data(asset_qs)
 
@@ -215,7 +232,7 @@
         asset_qs = Asset.objects.filter(
             item__project__campaign__slug=campaign_slug,
             transcription_status=TranscriptionStatus.COMPLETED,
-        )
+        ).order_by("item__project", "item", "sequence")
 
         assets = get_latest_transcription_data(asset_qs)

{"golden_diff": "diff --git a/exporter/views.py b/exporter/views.py\n--- a/exporter/views.py\n+++ b/exporter/views.py\n@@ -61,23 +61,40 @@\n     Uploads zip to S3 if configured.\n     \"\"\"\n \n+    # These assets should already be in the correct order - by item, seequence\n     for asset in assets:\n         asset_id = get_original_asset_id(asset.download_url)\n         logger.debug(\"Exporting asset %s into %s\", asset_id, export_base_dir)\n \n         asset_id = asset_id.replace(\":\", \"/\")\n         asset_path, asset_filename = os.path.split(asset_id)\n+        item_path, item_filename = os.path.split(asset_path)\n \n-        dest_path = os.path.join(export_base_dir, asset_path)\n-        os.makedirs(dest_path, exist_ok=True)\n+        asset_dest_path = os.path.join(export_base_dir, asset_path)\n+        os.makedirs(asset_dest_path, exist_ok=True)\n \n-        # Build transcription output text file\n-        text_output_path = os.path.join(dest_path, \"%s.txt\" % asset_filename)\n-        with open(text_output_path, \"w\") as f:\n+        # Build a transcription output text file for each asset\n+        asset_text_output_path = os.path.join(\n+            asset_dest_path, \"%s.txt\" % asset_filename\n+        )\n+        # Write the asset level transcription file\n+        with open(asset_text_output_path, \"w\") as f:\n+            f.write(asset.latest_transcription or \"\")\n+\n+        # Append this asset transcription to the item transcription\n+        item_text_output_path = os.path.join(asset_dest_path, \"%s.txt\" % item_filename)\n+        with open(item_text_output_path, \"a\") as f:\n             f.write(asset.latest_transcription or \"\")\n-            if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n-                f.write(\"\\n\\n\")\n-                f.write(settings.ATTRIBUTION_TEXT)\n+            f.write(\"\\n\\n\")\n+\n+    # Add attributions to the end of all text files found under asset_dest_path\n+    if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n+        for dirpath, dirnames, filenames in os.walk(export_base_dir, topdown=False):\n+            for each_text_file in (i for i in filenames if i.endswith(\".txt\")):\n+                this_text_file = os.path.join(dirpath, each_text_file)\n+                with open(this_text_file, \"a\") as f:\n+                    f.write(\"\\n\\n\")\n+                    f.write(settings.ATTRIBUTION_TEXT)\n \n     # Turn Structure into bagit format\n     bagit.make_bag(\n@@ -175,7 +192,7 @@\n             item__project__slug=project_slug,\n             item__item_id=item_id,\n             transcription_status=TranscriptionStatus.COMPLETED,\n-        )\n+        ).order_by(\"sequence\")\n \n         assets = get_latest_transcription_data(asset_qs)\n \n@@ -196,7 +213,7 @@\n             item__project__campaign__slug=campaign_slug,\n             item__project__slug=project_slug,\n             transcription_status=TranscriptionStatus.COMPLETED,\n-        )\n+        ).order_by(\"item\", \"sequence\")\n \n         assets = get_latest_transcription_data(asset_qs)\n \n@@ -215,7 +232,7 @@\n         asset_qs = Asset.objects.filter(\n             item__project__campaign__slug=campaign_slug,\n             transcription_status=TranscriptionStatus.COMPLETED,\n-        )\n+        ).order_by(\"item__project\", \"item\", \"sequence\")\n \n         assets = get_latest_transcription_data(asset_qs)\n", "issue": "Include one-page text per item in BagIt export\n**Is your feature request related to a problem? Please describe.**\r\nIn the BagIt export which feeds transcriptions into loc.gov, we want to provide one plain text file per item which has all the transcription text from all pages.\r\n\r\n**Describe the solution you'd like**\r\nThe text file should contain all pages in the item and be named with the item ID. There should be a single attribution at the bottom of all the text.\r\n\r\n\n", "before_files": [{"content": "import os\nimport re\nimport shutil\nimport tempfile\nfrom logging import getLogger\n\nimport bagit\nimport boto3\nfrom django.conf import settings\nfrom django.contrib.admin.views.decorators import staff_member_required\nfrom django.db.models import OuterRef, Subquery\nfrom django.http import HttpResponse, HttpResponseRedirect\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\nfrom tabular_export.core import export_to_csv_response, flatten_queryset\n\nfrom concordia.models import Asset, Transcription, TranscriptionStatus\n\nlogger = getLogger(__name__)\n\n\ndef get_latest_transcription_data(asset_qs):\n    latest_trans_subquery = (\n        Transcription.objects.filter(asset=OuterRef(\"pk\"))\n        .order_by(\"-pk\")\n        .values(\"text\")\n    )\n\n    assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))\n    return assets\n\n\ndef get_original_asset_id(download_url):\n    \"\"\"\n    Extract the bit from the download url\n    that identifies this image uniquely on loc.gov\n    \"\"\"\n    if download_url.startswith(\"http://tile.loc.gov/\"):\n        pattern = r\"/service:([A-Za-z0-9:\\-]+)/\"\n        asset_id = re.search(pattern, download_url)\n        if not asset_id:\n            logger.error(\n                \"Couldn't find a matching asset ID in download URL %s\", download_url\n            )\n            raise AssertionError\n        else:\n            matching_asset_id = asset_id.group(1)\n            logger.debug(\n                \"Found asset ID %s in download URL %s\", matching_asset_id, download_url\n            )\n            return matching_asset_id\n    else:\n        logger.warning(\"Download URL doesn't start with tile.loc.gov: %s\", download_url)\n        return download_url\n\n\ndef do_bagit_export(assets, export_base_dir, export_filename_base):\n    \"\"\"\n    Executes bagit.py to turn temp directory into LC-specific bagit strucutre.\n    Builds and exports bagit structure as zip.\n    Uploads zip to S3 if configured.\n    \"\"\"\n\n    for asset in assets:\n        asset_id = get_original_asset_id(asset.download_url)\n        logger.debug(\"Exporting asset %s into %s\", asset_id, export_base_dir)\n\n        asset_id = asset_id.replace(\":\", \"/\")\n        asset_path, asset_filename = os.path.split(asset_id)\n\n        dest_path = os.path.join(export_base_dir, asset_path)\n        os.makedirs(dest_path, exist_ok=True)\n\n        # Build transcription output text file\n        text_output_path = os.path.join(dest_path, \"%s.txt\" % asset_filename)\n        with open(text_output_path, \"w\") as f:\n            f.write(asset.latest_transcription or \"\")\n            if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n                f.write(\"\\n\\n\")\n                f.write(settings.ATTRIBUTION_TEXT)\n\n    # Turn Structure into bagit format\n    bagit.make_bag(\n        export_base_dir,\n        {\n            \"Content-Access\": \"web\",\n            \"Content-Custodian\": \"dcms\",\n            \"Content-Process\": \"crowdsourced\",\n            \"Content-Type\": \"textual\",\n            \"LC-Bag-Id\": export_filename_base,\n            \"LC-Items\": \"%d transcriptions\" % len(assets),\n            \"LC-Project\": \"gdccrowd\",\n            \"License-Information\": \"Public domain\",\n        },\n    )\n\n    # Build .zip file of bagit formatted Campaign Folder\n    archive_name = export_base_dir\n    shutil.make_archive(archive_name, \"zip\", export_base_dir)\n\n    export_filename = \"%s.zip\" % export_filename_base\n\n    # Upload zip to S3 bucket\n    s3_bucket = getattr(settings, \"EXPORT_S3_BUCKET_NAME\", None)\n\n    if s3_bucket:\n        logger.debug(\"Uploading exported bag to S3 bucket %s\", s3_bucket)\n        s3 = boto3.resource(\"s3\")\n        s3.Bucket(s3_bucket).upload_file(\n            \"%s.zip\" % export_base_dir, \"%s\" % export_filename\n        )\n\n        return HttpResponseRedirect(\n            \"https://%s.s3.amazonaws.com/%s\" % (s3_bucket, export_filename)\n        )\n    else:\n        # Download zip from local storage\n        with open(\"%s.zip\" % export_base_dir, \"rb\") as zip_file:\n            response = HttpResponse(zip_file, content_type=\"application/zip\")\n        response[\"Content-Disposition\"] = \"attachment; filename=%s\" % export_filename\n        return response\n\n\nclass ExportCampaignToCSV(TemplateView):\n    \"\"\"\n    Exports the most recent transcription for each asset in a campaign\n    \"\"\"\n\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=self.kwargs[\"campaign_slug\"]\n        )\n        assets = get_latest_transcription_data(asset_qs)\n\n        headers, data = flatten_queryset(\n            assets,\n            field_names=[\n                \"item__project__campaign__title\",\n                \"item__project__title\",\n                \"item__title\",\n                \"item__item_id\",\n                \"title\",\n                \"transcription_status\",\n                \"download_url\",\n                \"latest_transcription\",\n            ],\n            extra_verbose_names={\n                \"item__project__campaign__title\": \"Campaign\",\n                \"item__project__title\": \"Project\",\n                \"item__title\": \"Item\",\n                \"item__item_id\": \"ItemId\",\n                \"item_id\": \"ItemId\",\n                \"title\": \"Asset\",\n                \"transcription_status\": \"AssetStatus\",\n                \"download_url\": \"DownloadUrl\",\n                \"latest_transcription\": \"Transcription\",\n            },\n        )\n\n        return export_to_csv_response(\n            \"%s.csv\" % self.kwargs[\"campaign_slug\"], headers, data\n        )\n\n\nclass ExportItemToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        item_id = self.kwargs[\"item_id\"]\n\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            item__item_id=item_id,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s-%s\" % (campaign_slug, project_slug, item_id)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportProjectToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s\" % (campaign_slug, project_slug)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportCampaignToBagit(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s\" % (campaign_slug,)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n", "path": "exporter/views.py"}], "after_files": [{"content": "import os\nimport re\nimport shutil\nimport tempfile\nfrom logging import getLogger\n\nimport bagit\nimport boto3\nfrom django.conf import settings\nfrom django.contrib.admin.views.decorators import staff_member_required\nfrom django.db.models import OuterRef, Subquery\nfrom django.http import HttpResponse, HttpResponseRedirect\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\nfrom tabular_export.core import export_to_csv_response, flatten_queryset\n\nfrom concordia.models import Asset, Transcription, TranscriptionStatus\n\nlogger = getLogger(__name__)\n\n\ndef get_latest_transcription_data(asset_qs):\n    latest_trans_subquery = (\n        Transcription.objects.filter(asset=OuterRef(\"pk\"))\n        .order_by(\"-pk\")\n        .values(\"text\")\n    )\n\n    assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))\n    return assets\n\n\ndef get_original_asset_id(download_url):\n    \"\"\"\n    Extract the bit from the download url\n    that identifies this image uniquely on loc.gov\n    \"\"\"\n    if download_url.startswith(\"http://tile.loc.gov/\"):\n        pattern = r\"/service:([A-Za-z0-9:\\-]+)/\"\n        asset_id = re.search(pattern, download_url)\n        if not asset_id:\n            logger.error(\n                \"Couldn't find a matching asset ID in download URL %s\", download_url\n            )\n            raise AssertionError\n        else:\n            matching_asset_id = asset_id.group(1)\n            logger.debug(\n                \"Found asset ID %s in download URL %s\", matching_asset_id, download_url\n            )\n            return matching_asset_id\n    else:\n        logger.warning(\"Download URL doesn't start with tile.loc.gov: %s\", download_url)\n        return download_url\n\n\ndef do_bagit_export(assets, export_base_dir, export_filename_base):\n    \"\"\"\n    Executes bagit.py to turn temp directory into LC-specific bagit strucutre.\n    Builds and exports bagit structure as zip.\n    Uploads zip to S3 if configured.\n    \"\"\"\n\n    # These assets should already be in the correct order - by item, seequence\n    for asset in assets:\n        asset_id = get_original_asset_id(asset.download_url)\n        logger.debug(\"Exporting asset %s into %s\", asset_id, export_base_dir)\n\n        asset_id = asset_id.replace(\":\", \"/\")\n        asset_path, asset_filename = os.path.split(asset_id)\n        item_path, item_filename = os.path.split(asset_path)\n\n        asset_dest_path = os.path.join(export_base_dir, asset_path)\n        os.makedirs(asset_dest_path, exist_ok=True)\n\n        # Build a transcription output text file for each asset\n        asset_text_output_path = os.path.join(\n            asset_dest_path, \"%s.txt\" % asset_filename\n        )\n        # Write the asset level transcription file\n        with open(asset_text_output_path, \"w\") as f:\n            f.write(asset.latest_transcription or \"\")\n\n        # Append this asset transcription to the item transcription\n        item_text_output_path = os.path.join(asset_dest_path, \"%s.txt\" % item_filename)\n        with open(item_text_output_path, \"a\") as f:\n            f.write(asset.latest_transcription or \"\")\n            f.write(\"\\n\\n\")\n\n    # Add attributions to the end of all text files found under asset_dest_path\n    if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n        for dirpath, dirnames, filenames in os.walk(export_base_dir, topdown=False):\n            for each_text_file in (i for i in filenames if i.endswith(\".txt\")):\n                this_text_file = os.path.join(dirpath, each_text_file)\n                with open(this_text_file, \"a\") as f:\n                    f.write(\"\\n\\n\")\n                    f.write(settings.ATTRIBUTION_TEXT)\n\n    # Turn Structure into bagit format\n    bagit.make_bag(\n        export_base_dir,\n        {\n            \"Content-Access\": \"web\",\n            \"Content-Custodian\": \"dcms\",\n            \"Content-Process\": \"crowdsourced\",\n            \"Content-Type\": \"textual\",\n            \"LC-Bag-Id\": export_filename_base,\n            \"LC-Items\": \"%d transcriptions\" % len(assets),\n            \"LC-Project\": \"gdccrowd\",\n            \"License-Information\": \"Public domain\",\n        },\n    )\n\n    # Build .zip file of bagit formatted Campaign Folder\n    archive_name = export_base_dir\n    shutil.make_archive(archive_name, \"zip\", export_base_dir)\n\n    export_filename = \"%s.zip\" % export_filename_base\n\n    # Upload zip to S3 bucket\n    s3_bucket = getattr(settings, \"EXPORT_S3_BUCKET_NAME\", None)\n\n    if s3_bucket:\n        logger.debug(\"Uploading exported bag to S3 bucket %s\", s3_bucket)\n        s3 = boto3.resource(\"s3\")\n        s3.Bucket(s3_bucket).upload_file(\n            \"%s.zip\" % export_base_dir, \"%s\" % export_filename\n        )\n\n        return HttpResponseRedirect(\n            \"https://%s.s3.amazonaws.com/%s\" % (s3_bucket, export_filename)\n        )\n    else:\n        # Download zip from local storage\n        with open(\"%s.zip\" % export_base_dir, \"rb\") as zip_file:\n            response = HttpResponse(zip_file, content_type=\"application/zip\")\n        response[\"Content-Disposition\"] = \"attachment; filename=%s\" % export_filename\n        return response\n\n\nclass ExportCampaignToCSV(TemplateView):\n    \"\"\"\n    Exports the most recent transcription for each asset in a campaign\n    \"\"\"\n\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=self.kwargs[\"campaign_slug\"]\n        )\n        assets = get_latest_transcription_data(asset_qs)\n\n        headers, data = flatten_queryset(\n            assets,\n            field_names=[\n                \"item__project__campaign__title\",\n                \"item__project__title\",\n                \"item__title\",\n                \"item__item_id\",\n                \"title\",\n                \"transcription_status\",\n                \"download_url\",\n                \"latest_transcription\",\n            ],\n            extra_verbose_names={\n                \"item__project__campaign__title\": \"Campaign\",\n                \"item__project__title\": \"Project\",\n                \"item__title\": \"Item\",\n                \"item__item_id\": \"ItemId\",\n                \"item_id\": \"ItemId\",\n                \"title\": \"Asset\",\n                \"transcription_status\": \"AssetStatus\",\n                \"download_url\": \"DownloadUrl\",\n                \"latest_transcription\": \"Transcription\",\n            },\n        )\n\n        return export_to_csv_response(\n            \"%s.csv\" % self.kwargs[\"campaign_slug\"], headers, data\n        )\n\n\nclass ExportItemToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        item_id = self.kwargs[\"item_id\"]\n\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            item__item_id=item_id,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        ).order_by(\"sequence\")\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s-%s\" % (campaign_slug, project_slug, item_id)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportProjectToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        ).order_by(\"item\", \"sequence\")\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s\" % (campaign_slug, project_slug)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportCampaignToBagit(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        ).order_by(\"item__project\", \"item\", \"sequence\")\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s\" % (campaign_slug,)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n", "path": "exporter/views.py"}]}