We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Include attribution in exported text files
**Is your feature request related to a problem? Please describe.**
The plain text files, one per page / asset, which are exported in BagIt form need to have attributions at the bottom of the text.

**Describe the solution you'd like**
Each text file should contain the attribution "Transcribed and reviewed by volunteers participating in the By The People project at crowd.loc.gov." at the bottom.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `exporter/views.py`
Content:
```
1 import os
2 import re
3 import shutil
4 import tempfile
5 from logging import getLogger
6 
7 import bagit
8 import boto3
9 from django.conf import settings
10 from django.contrib.admin.views.decorators import staff_member_required
11 from django.db.models import OuterRef, Subquery
12 from django.http import HttpResponse, HttpResponseRedirect
13 from django.utils.decorators import method_decorator
14 from django.views.generic import TemplateView
15 from tabular_export.core import export_to_csv_response, flatten_queryset
16 
17 from concordia.models import Asset, Transcription, TranscriptionStatus
18 
19 logger = getLogger(__name__)
20 
21 
22 def get_latest_transcription_data(asset_qs):
23     latest_trans_subquery = (
24         Transcription.objects.filter(asset=OuterRef("pk"))
25         .order_by("-pk")
26         .values("text")
27     )
28 
29     assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))
30     return assets
31 
32 
33 def get_original_asset_id(download_url):
34     """
35     Extract the bit from the download url
36     that identifies this image uniquely on loc.gov
37     """
38     if download_url.startswith("http://tile.loc.gov/"):
39         pattern = r"/service:([A-Za-z0-9:\-]+)/"
40         asset_id = re.search(pattern, download_url)
41         if not asset_id:
42             logger.error(
43                 "Couldn't find a matching asset ID in download URL %s", download_url
44             )
45             raise AssertionError
46         else:
47             matching_asset_id = asset_id.group(1)
48             logger.debug(
49                 "Found asset ID %s in download URL %s", matching_asset_id, download_url
50             )
51             return matching_asset_id
52     else:
53         logger.warning("Download URL doesn't start with tile.loc.gov: %s", download_url)
54         return download_url
55 
56 
57 def do_bagit_export(assets, export_base_dir, export_filename_base):
58     """
59     Executes bagit.py to turn temp directory into LC-specific bagit strucutre.
60     Builds and exports bagit structure as zip.
61     Uploads zip to S3 if configured.
62     """
63 
64     for asset in assets:
65         asset_id = get_original_asset_id(asset.download_url)
66         logger.debug("Exporting asset %s into %s", asset_id, export_base_dir)
67 
68         asset_id = asset_id.replace(":", "/")
69         asset_path, asset_filename = os.path.split(asset_id)
70 
71         dest_path = os.path.join(export_base_dir, asset_path)
72         os.makedirs(dest_path, exist_ok=True)
73 
74         # Build transcription output text file
75         text_output_path = os.path.join(dest_path, "%s.txt" % asset_filename)
76         with open(text_output_path, "w") as f:
77             f.write(asset.latest_transcription or "")
78 
79     # Turn Structure into bagit format
80     bagit.make_bag(
81         export_base_dir,
82         {
83             "Content-Access": "web",
84             "Content-Custodian": "dcms",
85             "Content-Process": "crowdsourced",
86             "Content-Type": "textual",
87             "LC-Bag-Id": export_filename_base,
88             "LC-Items": "%d transcriptions" % len(assets),
89             "LC-Project": "gdccrowd",
90             "License-Information": "Public domain",
91         },
92     )
93 
94     # Build .zip file of bagit formatted Campaign Folder
95     archive_name = export_base_dir
96     shutil.make_archive(archive_name, "zip", export_base_dir)
97 
98     export_filename = "%s.zip" % export_filename_base
99 
100     # Upload zip to S3 bucket
101     s3_bucket = getattr(settings, "EXPORT_S3_BUCKET_NAME", None)
102 
103     if s3_bucket:
104         logger.debug("Uploading exported bag to S3 bucket %s", s3_bucket)
105         s3 = boto3.resource("s3")
106         s3.Bucket(s3_bucket).upload_file(
107             "%s.zip" % export_base_dir, "%s" % export_filename
108         )
109 
110         return HttpResponseRedirect(
111             "https://%s.s3.amazonaws.com/%s" % (s3_bucket, export_filename)
112         )
113     else:
114         # Download zip from local storage
115         with open("%s.zip" % export_base_dir, "rb") as zip_file:
116             response = HttpResponse(zip_file, content_type="application/zip")
117         response["Content-Disposition"] = "attachment; filename=%s" % export_filename
118         return response
119 
120 
121 class ExportCampaignToCSV(TemplateView):
122     """
123     Exports the most recent transcription for each asset in a campaign
124     """
125 
126     @method_decorator(staff_member_required)
127     def get(self, request, *args, **kwargs):
128         asset_qs = Asset.objects.filter(
129             item__project__campaign__slug=self.kwargs["campaign_slug"]
130         )
131         assets = get_latest_transcription_data(asset_qs)
132 
133         headers, data = flatten_queryset(
134             assets,
135             field_names=[
136                 "item__project__campaign__title",
137                 "item__project__title",
138                 "item__title",
139                 "item__item_id",
140                 "title",
141                 "transcription_status",
142                 "download_url",
143                 "latest_transcription",
144             ],
145             extra_verbose_names={
146                 "item__project__campaign__title": "Campaign",
147                 "item__project__title": "Project",
148                 "item__title": "Item",
149                 "item__item_id": "ItemId",
150                 "item_id": "ItemId",
151                 "title": "Asset",
152                 "transcription_status": "AssetStatus",
153                 "download_url": "DownloadUrl",
154                 "latest_transcription": "Transcription",
155             },
156         )
157 
158         return export_to_csv_response(
159             "%s.csv" % self.kwargs["campaign_slug"], headers, data
160         )
161 
162 
163 class ExportItemToBagIt(TemplateView):
164     @method_decorator(staff_member_required)
165     def get(self, request, *args, **kwargs):
166         campaign_slug = self.kwargs["campaign_slug"]
167         project_slug = self.kwargs["project_slug"]
168         item_id = self.kwargs["item_id"]
169 
170         asset_qs = Asset.objects.filter(
171             item__project__campaign__slug=campaign_slug,
172             item__project__slug=project_slug,
173             item__item_id=item_id,
174             transcription_status=TranscriptionStatus.COMPLETED,
175         )
176 
177         assets = get_latest_transcription_data(asset_qs)
178 
179         export_filename_base = "%s-%s-%s" % (campaign_slug, project_slug, item_id)
180 
181         with tempfile.TemporaryDirectory(
182             prefix=export_filename_base
183         ) as export_base_dir:
184             return do_bagit_export(assets, export_base_dir, export_filename_base)
185 
186 
187 class ExportProjectToBagIt(TemplateView):
188     @method_decorator(staff_member_required)
189     def get(self, request, *args, **kwargs):
190         campaign_slug = self.kwargs["campaign_slug"]
191         project_slug = self.kwargs["project_slug"]
192         asset_qs = Asset.objects.filter(
193             item__project__campaign__slug=campaign_slug,
194             item__project__slug=project_slug,
195             transcription_status=TranscriptionStatus.COMPLETED,
196         )
197 
198         assets = get_latest_transcription_data(asset_qs)
199 
200         export_filename_base = "%s-%s" % (campaign_slug, project_slug)
201 
202         with tempfile.TemporaryDirectory(
203             prefix=export_filename_base
204         ) as export_base_dir:
205             return do_bagit_export(assets, export_base_dir, export_filename_base)
206 
207 
208 class ExportCampaignToBagit(TemplateView):
209     @method_decorator(staff_member_required)
210     def get(self, request, *args, **kwargs):
211         campaign_slug = self.kwargs["campaign_slug"]
212         asset_qs = Asset.objects.filter(
213             item__project__campaign__slug=campaign_slug,
214             transcription_status=TranscriptionStatus.COMPLETED,
215         )
216 
217         assets = get_latest_transcription_data(asset_qs)
218 
219         export_filename_base = "%s" % (campaign_slug,)
220 
221         with tempfile.TemporaryDirectory(
222             prefix=export_filename_base
223         ) as export_base_dir:
224             return do_bagit_export(assets, export_base_dir, export_filename_base)
225
```

Path: `concordia/settings_prod.py`
Content:
```
1 import json
2 import os
3 
4 from .secrets import get_secret
5 from .settings_template import *
6 
7 LOGGING["handlers"]["stream"]["level"] = "INFO"
8 LOGGING["handlers"]["file"]["level"] = "INFO"
9 LOGGING["handlers"]["file"]["filename"] = "./logs/concordia-web.log"
10 LOGGING["handlers"]["celery"]["level"] = "INFO"
11 LOGGING["handlers"]["celery"]["filename"] = "./logs/concordia-celery.log"
12 LOGGING["loggers"]["django"]["level"] = "INFO"
13 LOGGING["loggers"]["celery"]["level"] = "INFO"
14 
15 if os.getenv("AWS"):
16     ENV_NAME = os.getenv("ENV_NAME")
17 
18     django_secret_json = get_secret("crowd/%s/Django/SecretKey" % ENV_NAME)
19     django_secret = json.loads(django_secret_json)
20     DJANGO_SECRET_KEY = django_secret["DjangoSecretKey"]
21 
22     postgres_secret_json = get_secret("crowd/%s/DB/MasterUserPassword" % ENV_NAME)
23     postgres_secret = json.loads(postgres_secret_json)
24 
25     DATABASES["default"].update({"PASSWORD": postgres_secret["password"]})
26 
27     smtp_secret_json = get_secret("concordia/SMTP")
28     smtp_secret = json.loads(smtp_secret_json)
29     EMAIL_HOST = smtp_secret["Hostname"]
30     EMAIL_HOST_USER = smtp_secret["Username"]
31     EMAIL_HOST_PASSWORD = smtp_secret["Password"]
32 
33 else:
34     DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "changeme")
35     EMAIL_HOST = os.environ.get("EMAIL_HOST", "localhost")
36     EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", "")
37     EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", "")
38 
39 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
40 
41 EMAIL_USE_TLS = True
42 EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
43 EMAIL_PORT = 587
44 DEFAULT_FROM_EMAIL = os.environ.get("DEFAULT_FROM_EMAIL", "crowd@loc.gov")
45 DEFAULT_TO_EMAIL = DEFAULT_FROM_EMAIL
46 
47 CSRF_COOKIE_SECURE = True
48 
49 CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "pyamqp://guest@rabbit:5672")
50 CELERY_RESULT_BACKEND = "rpc://"
51 
52 S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME")
53 EXPORT_S3_BUCKET_NAME = os.getenv("EXPORT_S3_BUCKET_NAME")
54 
55 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
56 AWS_STORAGE_BUCKET_NAME = S3_BUCKET_NAME
57 AWS_DEFAULT_ACL = None  # Don't set an ACL on the files, inherit the bucket ACLs
58 
59 if CONCORDIA_ENVIRONMENT == "production":
60     MEDIA_URL = "https://crowd-media.loc.gov/"
61 else:
62     MEDIA_URL = "https://%s.s3.amazonaws.com/" % S3_BUCKET_NAME
63 
64 ELASTICSEARCH_DSL_AUTOSYNC = os.getenv("ELASTICSEARCH_DSL_AUTOSYNC", False)
65 
66 INSTALLED_APPS += ["django_elasticsearch_dsl"]
67 
68 ELASTICSEARCH_DSL_SIGNAL_PROCESSOR = (
69     "django_elasticsearch_dsl.signals.RealTimeSignalProcessor"
70 )
71 ELASTICSEARCH_DSL = {
72     "default": {"hosts": os.getenv("ELASTICSEARCH_ENDPOINT", "elk:9200")}
73 }
74 
75 # HMAC activation flow provide the two-step registration process,
76 # the user signs up and then completes activation via email instructions.
77 
78 REGISTRATION_SALT = "django_registration"  # doesn't need to be secret
79 
80 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
81 
82 RATELIMIT_BLOCK = os.getenv("RATELIMIT_BLOCK", "").lower() not in ("false", "0")
83
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/concordia/settings_prod.py b/concordia/settings_prod.py
--- a/concordia/settings_prod.py
+++ b/concordia/settings_prod.py
@@ -80,3 +80,9 @@
 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
 
 RATELIMIT_BLOCK = os.getenv("RATELIMIT_BLOCK", "").lower() not in ("false", "0")
+
+# Exporter attribution text for BagIt exports to LC
+ATTRIBUTION_TEXT = (
+    "Transcribed and reviewed by volunteers participating in the "
+    "By The People project at crowd.loc.gov."
+)
diff --git a/exporter/views.py b/exporter/views.py
--- a/exporter/views.py
+++ b/exporter/views.py
@@ -75,6 +75,9 @@
         text_output_path = os.path.join(dest_path, "%s.txt" % asset_filename)
         with open(text_output_path, "w") as f:
             f.write(asset.latest_transcription or "")
+            if hasattr(settings, "ATTRIBUTION_TEXT"):
+                f.write("\n\n")
+                f.write(settings.ATTRIBUTION_TEXT)
 
     # Turn Structure into bagit format
     bagit.make_bag(

{"golden_diff": "diff --git a/concordia/settings_prod.py b/concordia/settings_prod.py\n--- a/concordia/settings_prod.py\n+++ b/concordia/settings_prod.py\n@@ -80,3 +80,9 @@\n SESSION_ENGINE = \"django.contrib.sessions.backends.cached_db\"\n \n RATELIMIT_BLOCK = os.getenv(\"RATELIMIT_BLOCK\", \"\").lower() not in (\"false\", \"0\")\n+\n+# Exporter attribution text for BagIt exports to LC\n+ATTRIBUTION_TEXT = (\n+    \"Transcribed and reviewed by volunteers participating in the \"\n+    \"By The People project at crowd.loc.gov.\"\n+)\ndiff --git a/exporter/views.py b/exporter/views.py\n--- a/exporter/views.py\n+++ b/exporter/views.py\n@@ -75,6 +75,9 @@\n         text_output_path = os.path.join(dest_path, \"%s.txt\" % asset_filename)\n         with open(text_output_path, \"w\") as f:\n             f.write(asset.latest_transcription or \"\")\n+            if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n+                f.write(\"\\n\\n\")\n+                f.write(settings.ATTRIBUTION_TEXT)\n \n     # Turn Structure into bagit format\n     bagit.make_bag(\n", "issue": "Include attribution in exported text files\n**Is your feature request related to a problem? Please describe.**\r\nThe plain text files, one per page / asset, which are exported in BagIt form need to have attributions at the bottom of the text.\r\n\r\n**Describe the solution you'd like**\r\nEach text file should contain the attribution \"Transcribed and reviewed by volunteers participating in the By The People project at crowd.loc.gov.\" at the bottom.\r\n\n", "before_files": [{"content": "import os\nimport re\nimport shutil\nimport tempfile\nfrom logging import getLogger\n\nimport bagit\nimport boto3\nfrom django.conf import settings\nfrom django.contrib.admin.views.decorators import staff_member_required\nfrom django.db.models import OuterRef, Subquery\nfrom django.http import HttpResponse, HttpResponseRedirect\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\nfrom tabular_export.core import export_to_csv_response, flatten_queryset\n\nfrom concordia.models import Asset, Transcription, TranscriptionStatus\n\nlogger = getLogger(__name__)\n\n\ndef get_latest_transcription_data(asset_qs):\n    latest_trans_subquery = (\n        Transcription.objects.filter(asset=OuterRef(\"pk\"))\n        .order_by(\"-pk\")\n        .values(\"text\")\n    )\n\n    assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))\n    return assets\n\n\ndef get_original_asset_id(download_url):\n    \"\"\"\n    Extract the bit from the download url\n    that identifies this image uniquely on loc.gov\n    \"\"\"\n    if download_url.startswith(\"http://tile.loc.gov/\"):\n        pattern = r\"/service:([A-Za-z0-9:\\-]+)/\"\n        asset_id = re.search(pattern, download_url)\n        if not asset_id:\n            logger.error(\n                \"Couldn't find a matching asset ID in download URL %s\", download_url\n            )\n            raise AssertionError\n        else:\n            matching_asset_id = asset_id.group(1)\n            logger.debug(\n                \"Found asset ID %s in download URL %s\", matching_asset_id, download_url\n            )\n            return matching_asset_id\n    else:\n        logger.warning(\"Download URL doesn't start with tile.loc.gov: %s\", download_url)\n        return download_url\n\n\ndef do_bagit_export(assets, export_base_dir, export_filename_base):\n    \"\"\"\n    Executes bagit.py to turn temp directory into LC-specific bagit strucutre.\n    Builds and exports bagit structure as zip.\n    Uploads zip to S3 if configured.\n    \"\"\"\n\n    for asset in assets:\n        asset_id = get_original_asset_id(asset.download_url)\n        logger.debug(\"Exporting asset %s into %s\", asset_id, export_base_dir)\n\n        asset_id = asset_id.replace(\":\", \"/\")\n        asset_path, asset_filename = os.path.split(asset_id)\n\n        dest_path = os.path.join(export_base_dir, asset_path)\n        os.makedirs(dest_path, exist_ok=True)\n\n        # Build transcription output text file\n        text_output_path = os.path.join(dest_path, \"%s.txt\" % asset_filename)\n        with open(text_output_path, \"w\") as f:\n            f.write(asset.latest_transcription or \"\")\n\n    # Turn Structure into bagit format\n    bagit.make_bag(\n        export_base_dir,\n        {\n            \"Content-Access\": \"web\",\n            \"Content-Custodian\": \"dcms\",\n            \"Content-Process\": \"crowdsourced\",\n            \"Content-Type\": \"textual\",\n            \"LC-Bag-Id\": export_filename_base,\n            \"LC-Items\": \"%d transcriptions\" % len(assets),\n            \"LC-Project\": \"gdccrowd\",\n            \"License-Information\": \"Public domain\",\n        },\n    )\n\n    # Build .zip file of bagit formatted Campaign Folder\n    archive_name = export_base_dir\n    shutil.make_archive(archive_name, \"zip\", export_base_dir)\n\n    export_filename = \"%s.zip\" % export_filename_base\n\n    # Upload zip to S3 bucket\n    s3_bucket = getattr(settings, \"EXPORT_S3_BUCKET_NAME\", None)\n\n    if s3_bucket:\n        logger.debug(\"Uploading exported bag to S3 bucket %s\", s3_bucket)\n        s3 = boto3.resource(\"s3\")\n        s3.Bucket(s3_bucket).upload_file(\n            \"%s.zip\" % export_base_dir, \"%s\" % export_filename\n        )\n\n        return HttpResponseRedirect(\n            \"https://%s.s3.amazonaws.com/%s\" % (s3_bucket, export_filename)\n        )\n    else:\n        # Download zip from local storage\n        with open(\"%s.zip\" % export_base_dir, \"rb\") as zip_file:\n            response = HttpResponse(zip_file, content_type=\"application/zip\")\n        response[\"Content-Disposition\"] = \"attachment; filename=%s\" % export_filename\n        return response\n\n\nclass ExportCampaignToCSV(TemplateView):\n    \"\"\"\n    Exports the most recent transcription for each asset in a campaign\n    \"\"\"\n\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=self.kwargs[\"campaign_slug\"]\n        )\n        assets = get_latest_transcription_data(asset_qs)\n\n        headers, data = flatten_queryset(\n            assets,\n            field_names=[\n                \"item__project__campaign__title\",\n                \"item__project__title\",\n                \"item__title\",\n                \"item__item_id\",\n                \"title\",\n                \"transcription_status\",\n                \"download_url\",\n                \"latest_transcription\",\n            ],\n            extra_verbose_names={\n                \"item__project__campaign__title\": \"Campaign\",\n                \"item__project__title\": \"Project\",\n                \"item__title\": \"Item\",\n                \"item__item_id\": \"ItemId\",\n                \"item_id\": \"ItemId\",\n                \"title\": \"Asset\",\n                \"transcription_status\": \"AssetStatus\",\n                \"download_url\": \"DownloadUrl\",\n                \"latest_transcription\": \"Transcription\",\n            },\n        )\n\n        return export_to_csv_response(\n            \"%s.csv\" % self.kwargs[\"campaign_slug\"], headers, data\n        )\n\n\nclass ExportItemToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        item_id = self.kwargs[\"item_id\"]\n\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            item__item_id=item_id,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s-%s\" % (campaign_slug, project_slug, item_id)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportProjectToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s\" % (campaign_slug, project_slug)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportCampaignToBagit(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s\" % (campaign_slug,)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n", "path": "exporter/views.py"}, {"content": "import json\nimport os\n\nfrom .secrets import get_secret\nfrom .settings_template import *\n\nLOGGING[\"handlers\"][\"stream\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"file\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"file\"][\"filename\"] = \"./logs/concordia-web.log\"\nLOGGING[\"handlers\"][\"celery\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"celery\"][\"filename\"] = \"./logs/concordia-celery.log\"\nLOGGING[\"loggers\"][\"django\"][\"level\"] = \"INFO\"\nLOGGING[\"loggers\"][\"celery\"][\"level\"] = \"INFO\"\n\nif os.getenv(\"AWS\"):\n    ENV_NAME = os.getenv(\"ENV_NAME\")\n\n    django_secret_json = get_secret(\"crowd/%s/Django/SecretKey\" % ENV_NAME)\n    django_secret = json.loads(django_secret_json)\n    DJANGO_SECRET_KEY = django_secret[\"DjangoSecretKey\"]\n\n    postgres_secret_json = get_secret(\"crowd/%s/DB/MasterUserPassword\" % ENV_NAME)\n    postgres_secret = json.loads(postgres_secret_json)\n\n    DATABASES[\"default\"].update({\"PASSWORD\": postgres_secret[\"password\"]})\n\n    smtp_secret_json = get_secret(\"concordia/SMTP\")\n    smtp_secret = json.loads(smtp_secret_json)\n    EMAIL_HOST = smtp_secret[\"Hostname\"]\n    EMAIL_HOST_USER = smtp_secret[\"Username\"]\n    EMAIL_HOST_PASSWORD = smtp_secret[\"Password\"]\n\nelse:\n    DJANGO_SECRET_KEY = os.getenv(\"DJANGO_SECRET_KEY\", \"changeme\")\n    EMAIL_HOST = os.environ.get(\"EMAIL_HOST\", \"localhost\")\n    EMAIL_HOST_USER = os.environ.get(\"EMAIL_HOST_USER\", \"\")\n    EMAIL_HOST_PASSWORD = os.environ.get(\"EMAIL_HOST_PASSWORD\", \"\")\n\nSECURE_PROXY_SSL_HEADER = (\"HTTP_X_FORWARDED_PROTO\", \"https\")\n\nEMAIL_USE_TLS = True\nEMAIL_BACKEND = \"django.core.mail.backends.smtp.EmailBackend\"\nEMAIL_PORT = 587\nDEFAULT_FROM_EMAIL = os.environ.get(\"DEFAULT_FROM_EMAIL\", \"crowd@loc.gov\")\nDEFAULT_TO_EMAIL = DEFAULT_FROM_EMAIL\n\nCSRF_COOKIE_SECURE = True\n\nCELERY_BROKER_URL = os.getenv(\"CELERY_BROKER_URL\", \"pyamqp://guest@rabbit:5672\")\nCELERY_RESULT_BACKEND = \"rpc://\"\n\nS3_BUCKET_NAME = os.getenv(\"S3_BUCKET_NAME\")\nEXPORT_S3_BUCKET_NAME = os.getenv(\"EXPORT_S3_BUCKET_NAME\")\n\nDEFAULT_FILE_STORAGE = \"storages.backends.s3boto3.S3Boto3Storage\"\nAWS_STORAGE_BUCKET_NAME = S3_BUCKET_NAME\nAWS_DEFAULT_ACL = None  # Don't set an ACL on the files, inherit the bucket ACLs\n\nif CONCORDIA_ENVIRONMENT == \"production\":\n    MEDIA_URL = \"https://crowd-media.loc.gov/\"\nelse:\n    MEDIA_URL = \"https://%s.s3.amazonaws.com/\" % S3_BUCKET_NAME\n\nELASTICSEARCH_DSL_AUTOSYNC = os.getenv(\"ELASTICSEARCH_DSL_AUTOSYNC\", False)\n\nINSTALLED_APPS += [\"django_elasticsearch_dsl\"]\n\nELASTICSEARCH_DSL_SIGNAL_PROCESSOR = (\n    \"django_elasticsearch_dsl.signals.RealTimeSignalProcessor\"\n)\nELASTICSEARCH_DSL = {\n    \"default\": {\"hosts\": os.getenv(\"ELASTICSEARCH_ENDPOINT\", \"elk:9200\")}\n}\n\n# HMAC activation flow provide the two-step registration process,\n# the user signs up and then completes activation via email instructions.\n\nREGISTRATION_SALT = \"django_registration\"  # doesn't need to be secret\n\nSESSION_ENGINE = \"django.contrib.sessions.backends.cached_db\"\n\nRATELIMIT_BLOCK = os.getenv(\"RATELIMIT_BLOCK\", \"\").lower() not in (\"false\", \"0\")\n", "path": "concordia/settings_prod.py"}], "after_files": [{"content": "import os\nimport re\nimport shutil\nimport tempfile\nfrom logging import getLogger\n\nimport bagit\nimport boto3\nfrom django.conf import settings\nfrom django.contrib.admin.views.decorators import staff_member_required\nfrom django.db.models import OuterRef, Subquery\nfrom django.http import HttpResponse, HttpResponseRedirect\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\nfrom tabular_export.core import export_to_csv_response, flatten_queryset\n\nfrom concordia.models import Asset, Transcription, TranscriptionStatus\n\nlogger = getLogger(__name__)\n\n\ndef get_latest_transcription_data(asset_qs):\n    latest_trans_subquery = (\n        Transcription.objects.filter(asset=OuterRef(\"pk\"))\n        .order_by(\"-pk\")\n        .values(\"text\")\n    )\n\n    assets = asset_qs.annotate(latest_transcription=Subquery(latest_trans_subquery[:1]))\n    return assets\n\n\ndef get_original_asset_id(download_url):\n    \"\"\"\n    Extract the bit from the download url\n    that identifies this image uniquely on loc.gov\n    \"\"\"\n    if download_url.startswith(\"http://tile.loc.gov/\"):\n        pattern = r\"/service:([A-Za-z0-9:\\-]+)/\"\n        asset_id = re.search(pattern, download_url)\n        if not asset_id:\n            logger.error(\n                \"Couldn't find a matching asset ID in download URL %s\", download_url\n            )\n            raise AssertionError\n        else:\n            matching_asset_id = asset_id.group(1)\n            logger.debug(\n                \"Found asset ID %s in download URL %s\", matching_asset_id, download_url\n            )\n            return matching_asset_id\n    else:\n        logger.warning(\"Download URL doesn't start with tile.loc.gov: %s\", download_url)\n        return download_url\n\n\ndef do_bagit_export(assets, export_base_dir, export_filename_base):\n    \"\"\"\n    Executes bagit.py to turn temp directory into LC-specific bagit strucutre.\n    Builds and exports bagit structure as zip.\n    Uploads zip to S3 if configured.\n    \"\"\"\n\n    for asset in assets:\n        asset_id = get_original_asset_id(asset.download_url)\n        logger.debug(\"Exporting asset %s into %s\", asset_id, export_base_dir)\n\n        asset_id = asset_id.replace(\":\", \"/\")\n        asset_path, asset_filename = os.path.split(asset_id)\n\n        dest_path = os.path.join(export_base_dir, asset_path)\n        os.makedirs(dest_path, exist_ok=True)\n\n        # Build transcription output text file\n        text_output_path = os.path.join(dest_path, \"%s.txt\" % asset_filename)\n        with open(text_output_path, \"w\") as f:\n            f.write(asset.latest_transcription or \"\")\n            if hasattr(settings, \"ATTRIBUTION_TEXT\"):\n                f.write(\"\\n\\n\")\n                f.write(settings.ATTRIBUTION_TEXT)\n\n    # Turn Structure into bagit format\n    bagit.make_bag(\n        export_base_dir,\n        {\n            \"Content-Access\": \"web\",\n            \"Content-Custodian\": \"dcms\",\n            \"Content-Process\": \"crowdsourced\",\n            \"Content-Type\": \"textual\",\n            \"LC-Bag-Id\": export_filename_base,\n            \"LC-Items\": \"%d transcriptions\" % len(assets),\n            \"LC-Project\": \"gdccrowd\",\n            \"License-Information\": \"Public domain\",\n        },\n    )\n\n    # Build .zip file of bagit formatted Campaign Folder\n    archive_name = export_base_dir\n    shutil.make_archive(archive_name, \"zip\", export_base_dir)\n\n    export_filename = \"%s.zip\" % export_filename_base\n\n    # Upload zip to S3 bucket\n    s3_bucket = getattr(settings, \"EXPORT_S3_BUCKET_NAME\", None)\n\n    if s3_bucket:\n        logger.debug(\"Uploading exported bag to S3 bucket %s\", s3_bucket)\n        s3 = boto3.resource(\"s3\")\n        s3.Bucket(s3_bucket).upload_file(\n            \"%s.zip\" % export_base_dir, \"%s\" % export_filename\n        )\n\n        return HttpResponseRedirect(\n            \"https://%s.s3.amazonaws.com/%s\" % (s3_bucket, export_filename)\n        )\n    else:\n        # Download zip from local storage\n        with open(\"%s.zip\" % export_base_dir, \"rb\") as zip_file:\n            response = HttpResponse(zip_file, content_type=\"application/zip\")\n        response[\"Content-Disposition\"] = \"attachment; filename=%s\" % export_filename\n        return response\n\n\nclass ExportCampaignToCSV(TemplateView):\n    \"\"\"\n    Exports the most recent transcription for each asset in a campaign\n    \"\"\"\n\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=self.kwargs[\"campaign_slug\"]\n        )\n        assets = get_latest_transcription_data(asset_qs)\n\n        headers, data = flatten_queryset(\n            assets,\n            field_names=[\n                \"item__project__campaign__title\",\n                \"item__project__title\",\n                \"item__title\",\n                \"item__item_id\",\n                \"title\",\n                \"transcription_status\",\n                \"download_url\",\n                \"latest_transcription\",\n            ],\n            extra_verbose_names={\n                \"item__project__campaign__title\": \"Campaign\",\n                \"item__project__title\": \"Project\",\n                \"item__title\": \"Item\",\n                \"item__item_id\": \"ItemId\",\n                \"item_id\": \"ItemId\",\n                \"title\": \"Asset\",\n                \"transcription_status\": \"AssetStatus\",\n                \"download_url\": \"DownloadUrl\",\n                \"latest_transcription\": \"Transcription\",\n            },\n        )\n\n        return export_to_csv_response(\n            \"%s.csv\" % self.kwargs[\"campaign_slug\"], headers, data\n        )\n\n\nclass ExportItemToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        item_id = self.kwargs[\"item_id\"]\n\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            item__item_id=item_id,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s-%s\" % (campaign_slug, project_slug, item_id)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportProjectToBagIt(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        project_slug = self.kwargs[\"project_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            item__project__slug=project_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s-%s\" % (campaign_slug, project_slug)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n\n\nclass ExportCampaignToBagit(TemplateView):\n    @method_decorator(staff_member_required)\n    def get(self, request, *args, **kwargs):\n        campaign_slug = self.kwargs[\"campaign_slug\"]\n        asset_qs = Asset.objects.filter(\n            item__project__campaign__slug=campaign_slug,\n            transcription_status=TranscriptionStatus.COMPLETED,\n        )\n\n        assets = get_latest_transcription_data(asset_qs)\n\n        export_filename_base = \"%s\" % (campaign_slug,)\n\n        with tempfile.TemporaryDirectory(\n            prefix=export_filename_base\n        ) as export_base_dir:\n            return do_bagit_export(assets, export_base_dir, export_filename_base)\n", "path": "exporter/views.py"}, {"content": "import json\nimport os\n\nfrom .secrets import get_secret\nfrom .settings_template import *\n\nLOGGING[\"handlers\"][\"stream\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"file\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"file\"][\"filename\"] = \"./logs/concordia-web.log\"\nLOGGING[\"handlers\"][\"celery\"][\"level\"] = \"INFO\"\nLOGGING[\"handlers\"][\"celery\"][\"filename\"] = \"./logs/concordia-celery.log\"\nLOGGING[\"loggers\"][\"django\"][\"level\"] = \"INFO\"\nLOGGING[\"loggers\"][\"celery\"][\"level\"] = \"INFO\"\n\nif os.getenv(\"AWS\"):\n    ENV_NAME = os.getenv(\"ENV_NAME\")\n\n    django_secret_json = get_secret(\"crowd/%s/Django/SecretKey\" % ENV_NAME)\n    django_secret = json.loads(django_secret_json)\n    DJANGO_SECRET_KEY = django_secret[\"DjangoSecretKey\"]\n\n    postgres_secret_json = get_secret(\"crowd/%s/DB/MasterUserPassword\" % ENV_NAME)\n    postgres_secret = json.loads(postgres_secret_json)\n\n    DATABASES[\"default\"].update({\"PASSWORD\": postgres_secret[\"password\"]})\n\n    smtp_secret_json = get_secret(\"concordia/SMTP\")\n    smtp_secret = json.loads(smtp_secret_json)\n    EMAIL_HOST = smtp_secret[\"Hostname\"]\n    EMAIL_HOST_USER = smtp_secret[\"Username\"]\n    EMAIL_HOST_PASSWORD = smtp_secret[\"Password\"]\n\nelse:\n    DJANGO_SECRET_KEY = os.getenv(\"DJANGO_SECRET_KEY\", \"changeme\")\n    EMAIL_HOST = os.environ.get(\"EMAIL_HOST\", \"localhost\")\n    EMAIL_HOST_USER = os.environ.get(\"EMAIL_HOST_USER\", \"\")\n    EMAIL_HOST_PASSWORD = os.environ.get(\"EMAIL_HOST_PASSWORD\", \"\")\n\nSECURE_PROXY_SSL_HEADER = (\"HTTP_X_FORWARDED_PROTO\", \"https\")\n\nEMAIL_USE_TLS = True\nEMAIL_BACKEND = \"django.core.mail.backends.smtp.EmailBackend\"\nEMAIL_PORT = 587\nDEFAULT_FROM_EMAIL = os.environ.get(\"DEFAULT_FROM_EMAIL\", \"crowd@loc.gov\")\nDEFAULT_TO_EMAIL = DEFAULT_FROM_EMAIL\n\nCSRF_COOKIE_SECURE = True\n\nCELERY_BROKER_URL = os.getenv(\"CELERY_BROKER_URL\", \"pyamqp://guest@rabbit:5672\")\nCELERY_RESULT_BACKEND = \"rpc://\"\n\nS3_BUCKET_NAME = os.getenv(\"S3_BUCKET_NAME\")\nEXPORT_S3_BUCKET_NAME = os.getenv(\"EXPORT_S3_BUCKET_NAME\")\n\nDEFAULT_FILE_STORAGE = \"storages.backends.s3boto3.S3Boto3Storage\"\nAWS_STORAGE_BUCKET_NAME = S3_BUCKET_NAME\nAWS_DEFAULT_ACL = None  # Don't set an ACL on the files, inherit the bucket ACLs\n\nif CONCORDIA_ENVIRONMENT == \"production\":\n    MEDIA_URL = \"https://crowd-media.loc.gov/\"\nelse:\n    MEDIA_URL = \"https://%s.s3.amazonaws.com/\" % S3_BUCKET_NAME\n\nELASTICSEARCH_DSL_AUTOSYNC = os.getenv(\"ELASTICSEARCH_DSL_AUTOSYNC\", False)\n\nINSTALLED_APPS += [\"django_elasticsearch_dsl\"]\n\nELASTICSEARCH_DSL_SIGNAL_PROCESSOR = (\n    \"django_elasticsearch_dsl.signals.RealTimeSignalProcessor\"\n)\nELASTICSEARCH_DSL = {\n    \"default\": {\"hosts\": os.getenv(\"ELASTICSEARCH_ENDPOINT\", \"elk:9200\")}\n}\n\n# HMAC activation flow provide the two-step registration process,\n# the user signs up and then completes activation via email instructions.\n\nREGISTRATION_SALT = \"django_registration\"  # doesn't need to be secret\n\nSESSION_ENGINE = \"django.contrib.sessions.backends.cached_db\"\n\nRATELIMIT_BLOCK = os.getenv(\"RATELIMIT_BLOCK\", \"\").lower() not in (\"false\", \"0\")\n\n# Exporter attribution text for BagIt exports to LC\nATTRIBUTION_TEXT = (\n    \"Transcribed and reviewed by volunteers participating in the \"\n    \"By The People project at crowd.loc.gov.\"\n)\n", "path": "concordia/settings_prod.py"}]}