We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Un-hard-code ensembl version
### Context

https://github.com/AlexsLemonade/refinebio/pull/1308

### Problem or idea

Ensembl broke their version endpoint so we hardcoded the correct version. They've since fixed their version endpoint, so we can go back to pulling the version from there instead of hard coding it.

### Solution or next step

Use ensembl's API to pull the latest version rather than always using version 43.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `foreman/data_refinery_foreman/surveyor/transcriptome_index.py`
Content:
```
1 import re
2 import urllib
3 
4 from abc import ABC
5 from django.utils import timezone
6 from typing import List, Dict
7 
8 from data_refinery_common.job_lookup import ProcessorPipeline, Downloaders
9 from data_refinery_common.logging import get_and_configure_logger
10 from data_refinery_common.models import (
11     OriginalFile,
12     SurveyJobKeyValue,
13 )
14 from data_refinery_foreman.surveyor import utils
15 from data_refinery_foreman.surveyor.external_source import ExternalSourceSurveyor
16 
17 logger = get_and_configure_logger(__name__)
18 
19 
20 MAIN_DIVISION_URL_TEMPLATE = "https://rest.ensembl.org/info/species?content-type=application/json"
21 DIVISION_URL_TEMPLATE = ("https://rest.ensembl.org/info/genomes/division/{division}"
22                          "?content-type=application/json")
23 
24 TRANSCRIPTOME_URL_TEMPLATE = ("ftp://ftp.{url_root}/fasta/{species_sub_dir}/dna/"
25                               "{filename_species}.{assembly}.dna.{schema_type}.fa.gz")
26 GTF_URL_TEMPLATE = ("ftp://ftp.{url_root}/gtf/{species_sub_dir}/"
27                     "{filename_species}.{assembly}.{assembly_version}.gtf.gz")
28 
29 
30 # For whatever reason the division in the download URL is shortened in
31 # a way that doesn't seem to be discoverable programmatically. I've
32 # therefore created this lookup map:
33 DIVISION_LOOKUP = {"EnsemblPlants": "plants",
34                    "EnsemblFungi": "fungi",
35                    "EnsemblBacteria": "bacteria",
36                    "EnsemblProtists": "protists",
37                    "EnsemblMetazoa": "metazoa"}
38 
39 
40 # Ensembl will periodically release updated versions of the
41 # assemblies.  All divisions other than the main one have identical
42 # release versions. These urls will return what the most recent
43 # release version is.
44 MAIN_RELEASE_URL = "https://rest.ensembl.org/info/software?content-type=application/json"
45 DIVISION_RELEASE_URL = "https://rest.ensembl.org/info/eg_version?content-type=application/json"
46 
47 
48 class EnsemblUrlBuilder(ABC):
49     """Generates URLs for different divisions of Ensembl.
50 
51     Each division of Ensembl has different conventions for its
52     URLs. The logic contained in the init method of this base class is
53     appropriate for most, but not all of the divisions. However, the
54     logic contained in the build_* methods of this class is
55     appropriate for all divisions.
56     """
57 
58     def __init__(self, species: Dict):
59         """Species is a Dict containing parsed JSON from the Division API."""
60         self.url_root = "ensemblgenomes.org/pub/release-{assembly_version}/{short_division}"
61         self.short_division = DIVISION_LOOKUP[species["division"]]
62         self.assembly = species["assembly_name"].replace(" ", "_")
63 
64         # For some reason the API is returning version 44, which doesn't seem to exist
65         # in the FTP servers: ftp://ftp.ensemblgenomes.org/pub/
66         # That's why the version is hardcoded below.
67         # self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()["version"]
68         self.assembly_version = '43'
69 
70         self.species_sub_dir = species["name"]
71         self.filename_species = species["name"].capitalize()
72 
73         # These fields aren't needed for the URL, but they vary between
74         # the two REST APIs.
75         self.scientific_name = species["name"].upper()
76         self.taxonomy_id = species["taxonomy_id"]
77 
78     def build_transcriptome_url(self) -> str:
79         url_root = self.url_root.format(assembly_version=self.assembly_version,
80                                         short_division=self.short_division)
81         url = TRANSCRIPTOME_URL_TEMPLATE.format(url_root=url_root,
82                                                 species_sub_dir=self.species_sub_dir,
83                                                 filename_species=self.filename_species,
84                                                 assembly=self.assembly,
85                                                 schema_type="primary_assembly")
86 
87         # If the primary_assembly is not available use toplevel instead.
88         try:
89             # Ancient unresolved bug. WTF python: https://bugs.python.org/issue27973
90             urllib.request.urlcleanup()
91             file_handle = urllib.request.urlopen(url)
92             file_handle.close()
93             urllib.request.urlcleanup()
94         except:
95             url = url.replace("primary_assembly", "toplevel")
96 
97         return url
98 
99     def build_gtf_url(self) -> str:
100         url_root = self.url_root.format(assembly_version=self.assembly_version,
101                                         short_division=self.short_division)
102         return GTF_URL_TEMPLATE.format(url_root=url_root,
103                                        species_sub_dir=self.species_sub_dir,
104                                        filename_species=self.filename_species,
105                                        assembly=self.assembly,
106                                        assembly_version=self.assembly_version)
107 
108 
109 class MainEnsemblUrlBuilder(EnsemblUrlBuilder):
110     """Special logic specific to the main Ensembl division.
111 
112     There is one Ensembl division which is just called Ensembl. This
113     is confusing so I refer to it as the main Ensembl division. It
114     follows the same general pattern as the rest of them for URLs, but
115     just not quite the same base URL structure. Also its REST API
116     returns JSON with similar data except with slightly different key
117     names.
118     """
119 
120     def __init__(self, species: Dict):
121         self.url_root = "ensembl.org/pub/release-{assembly_version}"
122         self.short_division = None
123         self.species_sub_dir = species["name"]
124         self.filename_species = species["name"].capitalize()
125         self.assembly = species["assembly"]
126         self.assembly_version = utils.requests_retry_session().get(
127             MAIN_RELEASE_URL).json()["release"]
128         self.scientific_name = self.filename_species.replace("_", " ")
129         self.taxonomy_id = species["taxon_id"]
130 
131 
132 class EnsemblProtistsUrlBuilder(EnsemblUrlBuilder):
133     """Special logic specific to the EnsemblProtists division.
134 
135     EnsemblProtists is special because the first letter of the species
136     name is always capitalized within the name of the file, instead of
137     only when there's not a collection subnested.
138     """
139 
140     def __init__(self, species: Dict):
141         super().__init__(species)
142         self.filename_species = species["species"].capitalize()
143 
144 
145 class EnsemblFungiUrlBuilder(EnsemblProtistsUrlBuilder):
146     """The EnsemblFungi URLs work the similarly to Protists division.
147 
148     EnsemblFungi is special because there is an assembly_name TIGR
149     which needs to be corrected to CADRE for some reason.
150     """
151 
152     def __init__(self, species: Dict):
153         super().__init__(species)
154         if self.assembly == "TIGR":
155             self.assembly = "CADRE"
156 
157 
158 def ensembl_url_builder_factory(species: Dict) -> EnsemblUrlBuilder:
159     """Returns instance of EnsemblUrlBuilder or one of its subclasses.
160 
161     The class of the returned object is based on the species' division.
162     """
163     if species["division"] == "EnsemblProtists":
164         return EnsemblProtistsUrlBuilder(species)
165     elif species["division"] == "EnsemblFungi":
166         return EnsemblFungiUrlBuilder(species)
167     elif species["division"] == "EnsemblVertebrates":
168         return MainEnsemblUrlBuilder(species)
169     else:
170         return EnsemblUrlBuilder(species)
171 
172 
173 class TranscriptomeIndexSurveyor(ExternalSourceSurveyor):
174     def source_type(self):
175         return Downloaders.TRANSCRIPTOME_INDEX.value
176 
177     def _clean_metadata(self, species: Dict) -> Dict:
178         """Removes fields from metadata which shouldn't be stored.
179 
180         Also cast any None values to str so they can be stored in the
181         database.
182         These fields shouldn't be stored because:
183         The taxonomy id is stored as fields on the Organism.
184         Aliases and groups are lists we don't need.
185         """
186         species.pop("taxon_id") if "taxon_id" in species else None
187         species.pop("taxonomy_id") if "taxonomy_id" in species else None
188         species.pop("aliases") if "aliases" in species else None
189         species.pop("groups") if "groups" in species else None
190 
191         # Cast to List since we're modifying the size of the dict
192         # while iterating over it
193         for k, v in list(species.items()):
194             if v is None:
195                 species.pop(k)
196             else:
197                 species[k] = str(v)
198 
199         return species
200 
201     def _generate_files(self, species: Dict) -> None:
202         url_builder = ensembl_url_builder_factory(species)
203         fasta_download_url = url_builder.build_transcriptome_url()
204         gtf_download_url = url_builder.build_gtf_url()
205         
206         platform_accession_code = species.pop("division")
207         self._clean_metadata(species)
208 
209         all_new_files = []
210 
211         fasta_filename = url_builder.filename_species + ".fa.gz"
212         original_file = OriginalFile()
213         original_file.source_filename = fasta_filename
214         original_file.source_url = fasta_download_url
215         original_file.is_archive = True
216         original_file.is_downloaded = False
217         original_file.save()
218         all_new_files.append(original_file)
219 
220         gtf_filename = url_builder.filename_species + ".gtf.gz"
221         original_file = OriginalFile()
222         original_file.source_filename = gtf_filename
223         original_file.source_url = gtf_download_url
224         original_file.is_archive = True
225         original_file.is_downloaded = False
226         original_file.save()
227         all_new_files.append(original_file)
228 
229         return all_new_files
230 
231     def survey(self, source_type=None) -> bool:
232         """
233         Surveying here is a bit different than discovering an experiment
234         and samples.
235         """
236         if source_type != "TRANSCRIPTOME_INDEX":
237             return False
238 
239         try:
240             species_files = self.discover_species()
241         except Exception:
242             logger.exception(("Exception caught while discovering species. "
243                               "Terminating survey job."),
244                              survey_job=self.survey_job.id)
245             return False
246 
247         try:
248             for specie_file_list in species_files:
249                 self.queue_downloader_job_for_original_files(specie_file_list,
250                                                              is_transcriptome=True)
251         except Exception:
252             logger.exception(("Failed to queue downloader jobs. "
253                               "Terminating survey job."),
254                              survey_job=self.survey_job.id)
255             return False
256 
257         return True
258 
259     def discover_species(self):
260         ensembl_division = (
261             SurveyJobKeyValue
262             .objects
263             .get(survey_job_id=self.survey_job.id,
264                  key__exact="ensembl_division")
265             .value
266         )
267 
268         logger.info("Surveying %s division of ensembl.",
269                     ensembl_division,
270                     survey_job=self.survey_job.id)
271 
272         # The main division has a different base URL for its REST API.
273         if ensembl_division == "Ensembl":
274             r = utils.requests_retry_session().get(MAIN_DIVISION_URL_TEMPLATE)
275 
276             # Yes I'm aware that specieses isn't a word. However I need to
277             # distinguish between a singlular species and multiple species.
278             specieses = r.json()["species"]
279         else:
280             r = utils.requests_retry_session().get(DIVISION_URL_TEMPLATE.format(division=ensembl_division))
281             specieses = r.json()
282 
283         try:
284             organism_name = SurveyJobKeyValue.objects.get(survey_job_id=self.survey_job.id,
285                                                           key__exact="organism_name").value
286             organism_name = organism_name.lower().replace(' ', "_")
287         except SurveyJobKeyValue.DoesNotExist:
288             organism_name = None
289 
290         all_new_species = []
291         if organism_name:
292             for species in specieses:
293                 # This key varies based on whether the division is the
294                 # main one or not... why couldn't they just make them
295                 # consistent?
296                 if ('species' in species and species['species'] == organism_name) \
297                    or ('name' in species and species['name'] == organism_name):
298                     all_new_species.append(self._generate_files(species))
299                     break
300         else:
301             for species in specieses:
302                 all_new_species.append(self._generate_files(species))
303 
304         if len(all_new_species) == 0:
305             logger.error("Unable to find any species!",
306                          ensembl_division=ensembl_division,
307                          organism_name=organism_name)
308 
309         return all_new_species
310
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/foreman/data_refinery_foreman/surveyor/transcriptome_index.py b/foreman/data_refinery_foreman/surveyor/transcriptome_index.py
--- a/foreman/data_refinery_foreman/surveyor/transcriptome_index.py
+++ b/foreman/data_refinery_foreman/surveyor/transcriptome_index.py
@@ -60,12 +60,7 @@
         self.url_root = "ensemblgenomes.org/pub/release-{assembly_version}/{short_division}"
         self.short_division = DIVISION_LOOKUP[species["division"]]
         self.assembly = species["assembly_name"].replace(" ", "_")
-
-        # For some reason the API is returning version 44, which doesn't seem to exist
-        # in the FTP servers: ftp://ftp.ensemblgenomes.org/pub/
-        # That's why the version is hardcoded below.
-        # self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()["version"]
-        self.assembly_version = '43'
+        self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()["version"]
 
         self.species_sub_dir = species["name"]
         self.filename_species = species["name"].capitalize()

{"golden_diff": "diff --git a/foreman/data_refinery_foreman/surveyor/transcriptome_index.py b/foreman/data_refinery_foreman/surveyor/transcriptome_index.py\n--- a/foreman/data_refinery_foreman/surveyor/transcriptome_index.py\n+++ b/foreman/data_refinery_foreman/surveyor/transcriptome_index.py\n@@ -60,12 +60,7 @@\n         self.url_root = \"ensemblgenomes.org/pub/release-{assembly_version}/{short_division}\"\n         self.short_division = DIVISION_LOOKUP[species[\"division\"]]\n         self.assembly = species[\"assembly_name\"].replace(\" \", \"_\")\n-\n-        # For some reason the API is returning version 44, which doesn't seem to exist\n-        # in the FTP servers: ftp://ftp.ensemblgenomes.org/pub/\n-        # That's why the version is hardcoded below.\n-        # self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()[\"version\"]\n-        self.assembly_version = '43'\n+        self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()[\"version\"]\n \n         self.species_sub_dir = species[\"name\"]\n         self.filename_species = species[\"name\"].capitalize()\n", "issue": "Un-hard-code ensembl version\n### Context\r\n\r\nhttps://github.com/AlexsLemonade/refinebio/pull/1308\r\n\r\n### Problem or idea\r\n\r\nEnsembl broke their version endpoint so we hardcoded the correct version. They've since fixed their version endpoint, so we can go back to pulling the version from there instead of hard coding it.\r\n\r\n### Solution or next step\r\n\r\nUse ensembl's API to pull the latest version rather than always using version 43.\n", "before_files": [{"content": "import re\nimport urllib\n\nfrom abc import ABC\nfrom django.utils import timezone\nfrom typing import List, Dict\n\nfrom data_refinery_common.job_lookup import ProcessorPipeline, Downloaders\nfrom data_refinery_common.logging import get_and_configure_logger\nfrom data_refinery_common.models import (\n    OriginalFile,\n    SurveyJobKeyValue,\n)\nfrom data_refinery_foreman.surveyor import utils\nfrom data_refinery_foreman.surveyor.external_source import ExternalSourceSurveyor\n\nlogger = get_and_configure_logger(__name__)\n\n\nMAIN_DIVISION_URL_TEMPLATE = \"https://rest.ensembl.org/info/species?content-type=application/json\"\nDIVISION_URL_TEMPLATE = (\"https://rest.ensembl.org/info/genomes/division/{division}\"\n                         \"?content-type=application/json\")\n\nTRANSCRIPTOME_URL_TEMPLATE = (\"ftp://ftp.{url_root}/fasta/{species_sub_dir}/dna/\"\n                              \"{filename_species}.{assembly}.dna.{schema_type}.fa.gz\")\nGTF_URL_TEMPLATE = (\"ftp://ftp.{url_root}/gtf/{species_sub_dir}/\"\n                    \"{filename_species}.{assembly}.{assembly_version}.gtf.gz\")\n\n\n# For whatever reason the division in the download URL is shortened in\n# a way that doesn't seem to be discoverable programmatically. I've\n# therefore created this lookup map:\nDIVISION_LOOKUP = {\"EnsemblPlants\": \"plants\",\n                   \"EnsemblFungi\": \"fungi\",\n                   \"EnsemblBacteria\": \"bacteria\",\n                   \"EnsemblProtists\": \"protists\",\n                   \"EnsemblMetazoa\": \"metazoa\"}\n\n\n# Ensembl will periodically release updated versions of the\n# assemblies.  All divisions other than the main one have identical\n# release versions. These urls will return what the most recent\n# release version is.\nMAIN_RELEASE_URL = \"https://rest.ensembl.org/info/software?content-type=application/json\"\nDIVISION_RELEASE_URL = \"https://rest.ensembl.org/info/eg_version?content-type=application/json\"\n\n\nclass EnsemblUrlBuilder(ABC):\n    \"\"\"Generates URLs for different divisions of Ensembl.\n\n    Each division of Ensembl has different conventions for its\n    URLs. The logic contained in the init method of this base class is\n    appropriate for most, but not all of the divisions. However, the\n    logic contained in the build_* methods of this class is\n    appropriate for all divisions.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        \"\"\"Species is a Dict containing parsed JSON from the Division API.\"\"\"\n        self.url_root = \"ensemblgenomes.org/pub/release-{assembly_version}/{short_division}\"\n        self.short_division = DIVISION_LOOKUP[species[\"division\"]]\n        self.assembly = species[\"assembly_name\"].replace(\" \", \"_\")\n\n        # For some reason the API is returning version 44, which doesn't seem to exist\n        # in the FTP servers: ftp://ftp.ensemblgenomes.org/pub/\n        # That's why the version is hardcoded below.\n        # self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()[\"version\"]\n        self.assembly_version = '43'\n\n        self.species_sub_dir = species[\"name\"]\n        self.filename_species = species[\"name\"].capitalize()\n\n        # These fields aren't needed for the URL, but they vary between\n        # the two REST APIs.\n        self.scientific_name = species[\"name\"].upper()\n        self.taxonomy_id = species[\"taxonomy_id\"]\n\n    def build_transcriptome_url(self) -> str:\n        url_root = self.url_root.format(assembly_version=self.assembly_version,\n                                        short_division=self.short_division)\n        url = TRANSCRIPTOME_URL_TEMPLATE.format(url_root=url_root,\n                                                species_sub_dir=self.species_sub_dir,\n                                                filename_species=self.filename_species,\n                                                assembly=self.assembly,\n                                                schema_type=\"primary_assembly\")\n\n        # If the primary_assembly is not available use toplevel instead.\n        try:\n            # Ancient unresolved bug. WTF python: https://bugs.python.org/issue27973\n            urllib.request.urlcleanup()\n            file_handle = urllib.request.urlopen(url)\n            file_handle.close()\n            urllib.request.urlcleanup()\n        except:\n            url = url.replace(\"primary_assembly\", \"toplevel\")\n\n        return url\n\n    def build_gtf_url(self) -> str:\n        url_root = self.url_root.format(assembly_version=self.assembly_version,\n                                        short_division=self.short_division)\n        return GTF_URL_TEMPLATE.format(url_root=url_root,\n                                       species_sub_dir=self.species_sub_dir,\n                                       filename_species=self.filename_species,\n                                       assembly=self.assembly,\n                                       assembly_version=self.assembly_version)\n\n\nclass MainEnsemblUrlBuilder(EnsemblUrlBuilder):\n    \"\"\"Special logic specific to the main Ensembl division.\n\n    There is one Ensembl division which is just called Ensembl. This\n    is confusing so I refer to it as the main Ensembl division. It\n    follows the same general pattern as the rest of them for URLs, but\n    just not quite the same base URL structure. Also its REST API\n    returns JSON with similar data except with slightly different key\n    names.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        self.url_root = \"ensembl.org/pub/release-{assembly_version}\"\n        self.short_division = None\n        self.species_sub_dir = species[\"name\"]\n        self.filename_species = species[\"name\"].capitalize()\n        self.assembly = species[\"assembly\"]\n        self.assembly_version = utils.requests_retry_session().get(\n            MAIN_RELEASE_URL).json()[\"release\"]\n        self.scientific_name = self.filename_species.replace(\"_\", \" \")\n        self.taxonomy_id = species[\"taxon_id\"]\n\n\nclass EnsemblProtistsUrlBuilder(EnsemblUrlBuilder):\n    \"\"\"Special logic specific to the EnsemblProtists division.\n\n    EnsemblProtists is special because the first letter of the species\n    name is always capitalized within the name of the file, instead of\n    only when there's not a collection subnested.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        super().__init__(species)\n        self.filename_species = species[\"species\"].capitalize()\n\n\nclass EnsemblFungiUrlBuilder(EnsemblProtistsUrlBuilder):\n    \"\"\"The EnsemblFungi URLs work the similarly to Protists division.\n\n    EnsemblFungi is special because there is an assembly_name TIGR\n    which needs to be corrected to CADRE for some reason.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        super().__init__(species)\n        if self.assembly == \"TIGR\":\n            self.assembly = \"CADRE\"\n\n\ndef ensembl_url_builder_factory(species: Dict) -> EnsemblUrlBuilder:\n    \"\"\"Returns instance of EnsemblUrlBuilder or one of its subclasses.\n\n    The class of the returned object is based on the species' division.\n    \"\"\"\n    if species[\"division\"] == \"EnsemblProtists\":\n        return EnsemblProtistsUrlBuilder(species)\n    elif species[\"division\"] == \"EnsemblFungi\":\n        return EnsemblFungiUrlBuilder(species)\n    elif species[\"division\"] == \"EnsemblVertebrates\":\n        return MainEnsemblUrlBuilder(species)\n    else:\n        return EnsemblUrlBuilder(species)\n\n\nclass TranscriptomeIndexSurveyor(ExternalSourceSurveyor):\n    def source_type(self):\n        return Downloaders.TRANSCRIPTOME_INDEX.value\n\n    def _clean_metadata(self, species: Dict) -> Dict:\n        \"\"\"Removes fields from metadata which shouldn't be stored.\n\n        Also cast any None values to str so they can be stored in the\n        database.\n        These fields shouldn't be stored because:\n        The taxonomy id is stored as fields on the Organism.\n        Aliases and groups are lists we don't need.\n        \"\"\"\n        species.pop(\"taxon_id\") if \"taxon_id\" in species else None\n        species.pop(\"taxonomy_id\") if \"taxonomy_id\" in species else None\n        species.pop(\"aliases\") if \"aliases\" in species else None\n        species.pop(\"groups\") if \"groups\" in species else None\n\n        # Cast to List since we're modifying the size of the dict\n        # while iterating over it\n        for k, v in list(species.items()):\n            if v is None:\n                species.pop(k)\n            else:\n                species[k] = str(v)\n\n        return species\n\n    def _generate_files(self, species: Dict) -> None:\n        url_builder = ensembl_url_builder_factory(species)\n        fasta_download_url = url_builder.build_transcriptome_url()\n        gtf_download_url = url_builder.build_gtf_url()\n        \n        platform_accession_code = species.pop(\"division\")\n        self._clean_metadata(species)\n\n        all_new_files = []\n\n        fasta_filename = url_builder.filename_species + \".fa.gz\"\n        original_file = OriginalFile()\n        original_file.source_filename = fasta_filename\n        original_file.source_url = fasta_download_url\n        original_file.is_archive = True\n        original_file.is_downloaded = False\n        original_file.save()\n        all_new_files.append(original_file)\n\n        gtf_filename = url_builder.filename_species + \".gtf.gz\"\n        original_file = OriginalFile()\n        original_file.source_filename = gtf_filename\n        original_file.source_url = gtf_download_url\n        original_file.is_archive = True\n        original_file.is_downloaded = False\n        original_file.save()\n        all_new_files.append(original_file)\n\n        return all_new_files\n\n    def survey(self, source_type=None) -> bool:\n        \"\"\"\n        Surveying here is a bit different than discovering an experiment\n        and samples.\n        \"\"\"\n        if source_type != \"TRANSCRIPTOME_INDEX\":\n            return False\n\n        try:\n            species_files = self.discover_species()\n        except Exception:\n            logger.exception((\"Exception caught while discovering species. \"\n                              \"Terminating survey job.\"),\n                             survey_job=self.survey_job.id)\n            return False\n\n        try:\n            for specie_file_list in species_files:\n                self.queue_downloader_job_for_original_files(specie_file_list,\n                                                             is_transcriptome=True)\n        except Exception:\n            logger.exception((\"Failed to queue downloader jobs. \"\n                              \"Terminating survey job.\"),\n                             survey_job=self.survey_job.id)\n            return False\n\n        return True\n\n    def discover_species(self):\n        ensembl_division = (\n            SurveyJobKeyValue\n            .objects\n            .get(survey_job_id=self.survey_job.id,\n                 key__exact=\"ensembl_division\")\n            .value\n        )\n\n        logger.info(\"Surveying %s division of ensembl.\",\n                    ensembl_division,\n                    survey_job=self.survey_job.id)\n\n        # The main division has a different base URL for its REST API.\n        if ensembl_division == \"Ensembl\":\n            r = utils.requests_retry_session().get(MAIN_DIVISION_URL_TEMPLATE)\n\n            # Yes I'm aware that specieses isn't a word. However I need to\n            # distinguish between a singlular species and multiple species.\n            specieses = r.json()[\"species\"]\n        else:\n            r = utils.requests_retry_session().get(DIVISION_URL_TEMPLATE.format(division=ensembl_division))\n            specieses = r.json()\n\n        try:\n            organism_name = SurveyJobKeyValue.objects.get(survey_job_id=self.survey_job.id,\n                                                          key__exact=\"organism_name\").value\n            organism_name = organism_name.lower().replace(' ', \"_\")\n        except SurveyJobKeyValue.DoesNotExist:\n            organism_name = None\n\n        all_new_species = []\n        if organism_name:\n            for species in specieses:\n                # This key varies based on whether the division is the\n                # main one or not... why couldn't they just make them\n                # consistent?\n                if ('species' in species and species['species'] == organism_name) \\\n                   or ('name' in species and species['name'] == organism_name):\n                    all_new_species.append(self._generate_files(species))\n                    break\n        else:\n            for species in specieses:\n                all_new_species.append(self._generate_files(species))\n\n        if len(all_new_species) == 0:\n            logger.error(\"Unable to find any species!\",\n                         ensembl_division=ensembl_division,\n                         organism_name=organism_name)\n\n        return all_new_species\n", "path": "foreman/data_refinery_foreman/surveyor/transcriptome_index.py"}], "after_files": [{"content": "import re\nimport urllib\n\nfrom abc import ABC\nfrom django.utils import timezone\nfrom typing import List, Dict\n\nfrom data_refinery_common.job_lookup import ProcessorPipeline, Downloaders\nfrom data_refinery_common.logging import get_and_configure_logger\nfrom data_refinery_common.models import (\n    OriginalFile,\n    SurveyJobKeyValue,\n)\nfrom data_refinery_foreman.surveyor import utils\nfrom data_refinery_foreman.surveyor.external_source import ExternalSourceSurveyor\n\nlogger = get_and_configure_logger(__name__)\n\n\nMAIN_DIVISION_URL_TEMPLATE = \"https://rest.ensembl.org/info/species?content-type=application/json\"\nDIVISION_URL_TEMPLATE = (\"https://rest.ensembl.org/info/genomes/division/{division}\"\n                         \"?content-type=application/json\")\n\nTRANSCRIPTOME_URL_TEMPLATE = (\"ftp://ftp.{url_root}/fasta/{species_sub_dir}/dna/\"\n                              \"{filename_species}.{assembly}.dna.{schema_type}.fa.gz\")\nGTF_URL_TEMPLATE = (\"ftp://ftp.{url_root}/gtf/{species_sub_dir}/\"\n                    \"{filename_species}.{assembly}.{assembly_version}.gtf.gz\")\n\n\n# For whatever reason the division in the download URL is shortened in\n# a way that doesn't seem to be discoverable programmatically. I've\n# therefore created this lookup map:\nDIVISION_LOOKUP = {\"EnsemblPlants\": \"plants\",\n                   \"EnsemblFungi\": \"fungi\",\n                   \"EnsemblBacteria\": \"bacteria\",\n                   \"EnsemblProtists\": \"protists\",\n                   \"EnsemblMetazoa\": \"metazoa\"}\n\n\n# Ensembl will periodically release updated versions of the\n# assemblies.  All divisions other than the main one have identical\n# release versions. These urls will return what the most recent\n# release version is.\nMAIN_RELEASE_URL = \"https://rest.ensembl.org/info/software?content-type=application/json\"\nDIVISION_RELEASE_URL = \"https://rest.ensembl.org/info/eg_version?content-type=application/json\"\n\n\nclass EnsemblUrlBuilder(ABC):\n    \"\"\"Generates URLs for different divisions of Ensembl.\n\n    Each division of Ensembl has different conventions for its\n    URLs. The logic contained in the init method of this base class is\n    appropriate for most, but not all of the divisions. However, the\n    logic contained in the build_* methods of this class is\n    appropriate for all divisions.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        \"\"\"Species is a Dict containing parsed JSON from the Division API.\"\"\"\n        self.url_root = \"ensemblgenomes.org/pub/release-{assembly_version}/{short_division}\"\n        self.short_division = DIVISION_LOOKUP[species[\"division\"]]\n        self.assembly = species[\"assembly_name\"].replace(\" \", \"_\")\n        self.assembly_version = utils.requests_retry_session().get(DIVISION_RELEASE_URL).json()[\"version\"]\n\n        self.species_sub_dir = species[\"name\"]\n        self.filename_species = species[\"name\"].capitalize()\n\n        # These fields aren't needed for the URL, but they vary between\n        # the two REST APIs.\n        self.scientific_name = species[\"name\"].upper()\n        self.taxonomy_id = species[\"taxonomy_id\"]\n\n    def build_transcriptome_url(self) -> str:\n        url_root = self.url_root.format(assembly_version=self.assembly_version,\n                                        short_division=self.short_division)\n        url = TRANSCRIPTOME_URL_TEMPLATE.format(url_root=url_root,\n                                                species_sub_dir=self.species_sub_dir,\n                                                filename_species=self.filename_species,\n                                                assembly=self.assembly,\n                                                schema_type=\"primary_assembly\")\n\n        # If the primary_assembly is not available use toplevel instead.\n        try:\n            # Ancient unresolved bug. WTF python: https://bugs.python.org/issue27973\n            urllib.request.urlcleanup()\n            file_handle = urllib.request.urlopen(url)\n            file_handle.close()\n            urllib.request.urlcleanup()\n        except:\n            url = url.replace(\"primary_assembly\", \"toplevel\")\n\n        return url\n\n    def build_gtf_url(self) -> str:\n        url_root = self.url_root.format(assembly_version=self.assembly_version,\n                                        short_division=self.short_division)\n        return GTF_URL_TEMPLATE.format(url_root=url_root,\n                                       species_sub_dir=self.species_sub_dir,\n                                       filename_species=self.filename_species,\n                                       assembly=self.assembly,\n                                       assembly_version=self.assembly_version)\n\n\nclass MainEnsemblUrlBuilder(EnsemblUrlBuilder):\n    \"\"\"Special logic specific to the main Ensembl division.\n\n    There is one Ensembl division which is just called Ensembl. This\n    is confusing so I refer to it as the main Ensembl division. It\n    follows the same general pattern as the rest of them for URLs, but\n    just not quite the same base URL structure. Also its REST API\n    returns JSON with similar data except with slightly different key\n    names.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        self.url_root = \"ensembl.org/pub/release-{assembly_version}\"\n        self.short_division = None\n        self.species_sub_dir = species[\"name\"]\n        self.filename_species = species[\"name\"].capitalize()\n        self.assembly = species[\"assembly\"]\n        self.assembly_version = utils.requests_retry_session().get(\n            MAIN_RELEASE_URL).json()[\"release\"]\n        self.scientific_name = self.filename_species.replace(\"_\", \" \")\n        self.taxonomy_id = species[\"taxon_id\"]\n\n\nclass EnsemblProtistsUrlBuilder(EnsemblUrlBuilder):\n    \"\"\"Special logic specific to the EnsemblProtists division.\n\n    EnsemblProtists is special because the first letter of the species\n    name is always capitalized within the name of the file, instead of\n    only when there's not a collection subnested.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        super().__init__(species)\n        self.filename_species = species[\"species\"].capitalize()\n\n\nclass EnsemblFungiUrlBuilder(EnsemblProtistsUrlBuilder):\n    \"\"\"The EnsemblFungi URLs work the similarly to Protists division.\n\n    EnsemblFungi is special because there is an assembly_name TIGR\n    which needs to be corrected to CADRE for some reason.\n    \"\"\"\n\n    def __init__(self, species: Dict):\n        super().__init__(species)\n        if self.assembly == \"TIGR\":\n            self.assembly = \"CADRE\"\n\n\ndef ensembl_url_builder_factory(species: Dict) -> EnsemblUrlBuilder:\n    \"\"\"Returns instance of EnsemblUrlBuilder or one of its subclasses.\n\n    The class of the returned object is based on the species' division.\n    \"\"\"\n    if species[\"division\"] == \"EnsemblProtists\":\n        return EnsemblProtistsUrlBuilder(species)\n    elif species[\"division\"] == \"EnsemblFungi\":\n        return EnsemblFungiUrlBuilder(species)\n    elif species[\"division\"] == \"EnsemblVertebrates\":\n        return MainEnsemblUrlBuilder(species)\n    else:\n        return EnsemblUrlBuilder(species)\n\n\nclass TranscriptomeIndexSurveyor(ExternalSourceSurveyor):\n    def source_type(self):\n        return Downloaders.TRANSCRIPTOME_INDEX.value\n\n    def _clean_metadata(self, species: Dict) -> Dict:\n        \"\"\"Removes fields from metadata which shouldn't be stored.\n\n        Also cast any None values to str so they can be stored in the\n        database.\n        These fields shouldn't be stored because:\n        The taxonomy id is stored as fields on the Organism.\n        Aliases and groups are lists we don't need.\n        \"\"\"\n        species.pop(\"taxon_id\") if \"taxon_id\" in species else None\n        species.pop(\"taxonomy_id\") if \"taxonomy_id\" in species else None\n        species.pop(\"aliases\") if \"aliases\" in species else None\n        species.pop(\"groups\") if \"groups\" in species else None\n\n        # Cast to List since we're modifying the size of the dict\n        # while iterating over it\n        for k, v in list(species.items()):\n            if v is None:\n                species.pop(k)\n            else:\n                species[k] = str(v)\n\n        return species\n\n    def _generate_files(self, species: Dict) -> None:\n        url_builder = ensembl_url_builder_factory(species)\n        fasta_download_url = url_builder.build_transcriptome_url()\n        gtf_download_url = url_builder.build_gtf_url()\n        \n        platform_accession_code = species.pop(\"division\")\n        self._clean_metadata(species)\n\n        all_new_files = []\n\n        fasta_filename = url_builder.filename_species + \".fa.gz\"\n        original_file = OriginalFile()\n        original_file.source_filename = fasta_filename\n        original_file.source_url = fasta_download_url\n        original_file.is_archive = True\n        original_file.is_downloaded = False\n        original_file.save()\n        all_new_files.append(original_file)\n\n        gtf_filename = url_builder.filename_species + \".gtf.gz\"\n        original_file = OriginalFile()\n        original_file.source_filename = gtf_filename\n        original_file.source_url = gtf_download_url\n        original_file.is_archive = True\n        original_file.is_downloaded = False\n        original_file.save()\n        all_new_files.append(original_file)\n\n        return all_new_files\n\n    def survey(self, source_type=None) -> bool:\n        \"\"\"\n        Surveying here is a bit different than discovering an experiment\n        and samples.\n        \"\"\"\n        if source_type != \"TRANSCRIPTOME_INDEX\":\n            return False\n\n        try:\n            species_files = self.discover_species()\n        except Exception:\n            logger.exception((\"Exception caught while discovering species. \"\n                              \"Terminating survey job.\"),\n                             survey_job=self.survey_job.id)\n            return False\n\n        try:\n            for specie_file_list in species_files:\n                self.queue_downloader_job_for_original_files(specie_file_list,\n                                                             is_transcriptome=True)\n        except Exception:\n            logger.exception((\"Failed to queue downloader jobs. \"\n                              \"Terminating survey job.\"),\n                             survey_job=self.survey_job.id)\n            return False\n\n        return True\n\n    def discover_species(self):\n        ensembl_division = (\n            SurveyJobKeyValue\n            .objects\n            .get(survey_job_id=self.survey_job.id,\n                 key__exact=\"ensembl_division\")\n            .value\n        )\n\n        logger.info(\"Surveying %s division of ensembl.\",\n                    ensembl_division,\n                    survey_job=self.survey_job.id)\n\n        # The main division has a different base URL for its REST API.\n        if ensembl_division == \"Ensembl\":\n            r = utils.requests_retry_session().get(MAIN_DIVISION_URL_TEMPLATE)\n\n            # Yes I'm aware that specieses isn't a word. However I need to\n            # distinguish between a singlular species and multiple species.\n            specieses = r.json()[\"species\"]\n        else:\n            r = utils.requests_retry_session().get(DIVISION_URL_TEMPLATE.format(division=ensembl_division))\n            specieses = r.json()\n\n        try:\n            organism_name = SurveyJobKeyValue.objects.get(survey_job_id=self.survey_job.id,\n                                                          key__exact=\"organism_name\").value\n            organism_name = organism_name.lower().replace(' ', \"_\")\n        except SurveyJobKeyValue.DoesNotExist:\n            organism_name = None\n\n        all_new_species = []\n        if organism_name:\n            for species in specieses:\n                # This key varies based on whether the division is the\n                # main one or not... why couldn't they just make them\n                # consistent?\n                if ('species' in species and species['species'] == organism_name) \\\n                   or ('name' in species and species['name'] == organism_name):\n                    all_new_species.append(self._generate_files(species))\n                    break\n        else:\n            for species in specieses:\n                all_new_species.append(self._generate_files(species))\n\n        if len(all_new_species) == 0:\n            logger.error(\"Unable to find any species!\",\n                         ensembl_division=ensembl_division,\n                         organism_name=organism_name)\n\n        return all_new_species\n", "path": "foreman/data_refinery_foreman/surveyor/transcriptome_index.py"}]}