We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
check global quota in running RSESelector
Motivation
----------
check global quota not only when initializing the RSESelector but also when calling the select_rse method.


Modification
------------
update select_rse/__update_quota method in the RSESelector



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lib/rucio/core/rse_selector.py`
Content:
```
1 # Copyright European Organization for Nuclear Research (CERN)
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # You may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Authors:
9 # - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
10 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2019
11 # - Robert Illingworth, <illingwo@fnal.gov>, 2019
12 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
13 # - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019
14 #
15 # PY3K COMPATIBLE
16 
17 from random import uniform, shuffle
18 
19 from rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs, InvalidRuleWeight, RSEOverQuota
20 from rucio.core.account import has_account_attribute, get_usage, get_all_rse_usages_per_account
21 from rucio.core.account_limit import get_local_account_limit, get_global_account_limits
22 from rucio.core.rse import list_rse_attributes, has_rse_attribute, get_rse_limits
23 from rucio.core.rse_counter import get_counter as get_rse_counter
24 from rucio.db.sqla.session import read_session
25 
26 
27 class RSESelector():
28     """
29     Representation of the RSE selector
30     """
31 
32     @read_session
33     def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None):
34         """
35         Initialize the RSE Selector.
36 
37         :param account:               Account owning the rule.
38         :param rses:                  List of rse dictionaries.
39         :param weight:                Weighting to use.
40         :param copies:                Number of copies to create.
41         :param ignore_account_limit:  Flag if the quota should be ignored.
42         :param session:               DB Session in use.
43         :raises:                      InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs
44         """
45         self.account = account
46         self.rses = []  # [{'rse_id':, 'weight':, 'staging_area'}]
47         self.copies = copies
48         if weight is not None:
49             for rse in rses:
50                 attributes = list_rse_attributes(rse_id=rse['id'], session=session)
51                 availability_write = True if rse.get('availability', 7) & 2 else False
52                 if weight not in attributes:
53                     continue  # The RSE does not have the required weight set, therefore it is ignored
54                 try:
55                     self.rses.append({'rse_id': rse['id'],
56                                       'weight': float(attributes[weight]),
57                                       'mock_rse': attributes.get('mock', False),
58                                       'availability_write': availability_write,
59                                       'staging_area': rse['staging_area']})
60                 except ValueError:
61                     raise InvalidRuleWeight('The RSE \'%s\' has a non-number specified for the weight \'%s\'' % (rse['rse'], weight))
62         else:
63             for rse in rses:
64                 mock_rse = has_rse_attribute(rse['id'], 'mock', session=session)
65                 availability_write = True if rse.get('availability', 7) & 2 else False
66                 self.rses.append({'rse_id': rse['id'],
67                                   'weight': 1,
68                                   'mock_rse': mock_rse,
69                                   'availability_write': availability_write,
70                                   'staging_area': rse['staging_area']})
71 
72         if len(self.rses) < self.copies:
73             raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses)))
74 
75         rses_with_enough_quota = []
76         if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit:
77             for rse in self.rses:
78                 rse['quota_left'] = float('inf')
79                 rse['space_left'] = float('inf')
80                 rses_with_enough_quota.append(rse)
81         else:
82             global_quota_limit = get_global_account_limits(account=account, session=session)
83             all_rse_usages = {usage['rse_id']: usage['bytes'] for usage in get_all_rse_usages_per_account(account=account, session=session)}
84             for rse in self.rses:
85                 if rse['mock_rse']:
86                     rse['quota_left'] = float('inf')
87                     rse['space_left'] = float('inf')
88                     rses_with_enough_quota.append(rse)
89                 else:
90                     # check local quota
91                     local_quota_left = None
92                     quota_limit = get_local_account_limit(account=account, rse_id=rse['rse_id'], session=session)
93                     if quota_limit is None:
94                         local_quota_left = 0
95                     else:
96                         local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']
97                     # check global quota
98                     all_global_quota_enough = True
99                     for rse_expression, limit in global_quota_limit.items():
100                         if rse['rse_id'] in limit['resolved_rse_ids']:
101                             quota_limit = limit['limit']
102                             global_quota_left = None
103                             if quota_limit is None:
104                                 global_quota_left = 0
105                             else:
106                                 rse_expression_usage = 0
107                                 for rse_id in limit['resolved_rse_ids']:
108                                     rse_expression_usage += all_rse_usages.get(rse_id, 0)
109                                 global_quota_left = quota_limit - rse_expression_usage
110                             if global_quota_left <= 0:
111                                 all_global_quota_enough = False
112                                 break
113                     if local_quota_left > 0 and all_global_quota_enough:
114                         rse['quota_left'] = local_quota_left
115                         space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')
116                         if space_limit is None or space_limit < 0:
117                             rse['space_left'] = float('inf')
118                         else:
119                             rse['space_left'] = space_limit - get_rse_counter(rse_id=rse['rse_id'], session=session)['bytes']
120                         rses_with_enough_quota.append(rse)
121 
122         self.rses = rses_with_enough_quota
123         if len(self.rses) < self.copies:
124             raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
125 
126         # don't consider removing rses based on the total space here - because files already on the RSE are taken into account
127         # it is possible to have no space but still be able to fulfil the rule
128 
129     def select_rse(self, size, preferred_rse_ids, copies=0, blacklist=[], prioritize_order_over_weight=False, existing_rse_size=None):
130         """
131         Select n RSEs to replicate data to.
132 
133         :param size:                         Size of the block being replicated.
134         :param preferred_rse_ids:            Ordered list of preferred rses. (If possible replicate to them)
135         :param copies:                       Select this amount of copies, if 0 use the pre-defined rule value.
136         :param blacklist:                    List of blacklisted rses. (Do not put replicas on these sites)
137         :param prioritze_order_over_weight:  Prioritize the order of the preferred_rse_ids list over the picking done by weight.
138         :existing_rse_size:                  Dictionary of size of files already present at each rse
139         :returns:                            List of (RSE_id, staging_area, availability_write) tuples.
140         :raises:                             InsufficientAccountLimit, InsufficientTargetRSEs
141         """
142 
143         result = []
144         rses = self.rses
145         count = self.copies if copies == 0 else copies
146 
147         # Remove blacklisted rses
148         if blacklist:
149             rses = [rse for rse in self.rses if rse['rse_id'] not in blacklist]
150         if len(rses) < count:
151             raise InsufficientTargetRSEs('There are not enough target RSEs to fulfil the request at this time.')
152 
153         # Remove rses which do not have enough space, accounting for the files already at each rse
154         if existing_rse_size is None:
155             existing_rse_size = {}
156         rses = [rse for rse in rses if rse['space_left'] >= size - existing_rse_size.get(rse['rse_id'], 0)]
157         if len(rses) < count:
158             raise RSEOverQuota('There is insufficient space on any of the target RSE\'s to fullfill the operation.')
159 
160         # Remove rses which do not have enough quota
161         rses = [rse for rse in rses if rse['quota_left'] > size]
162         if len(rses) < count:
163             raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
164 
165         for copy in range(count):
166             # Remove rses already in the result set
167             rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]
168             rses_dict = {}
169             for rse in rses:
170                 rses_dict[rse['rse_id']] = rse
171             # Prioritize the preffered rses
172             preferred_rses = [rses_dict[rse_id] for rse_id in preferred_rse_ids if rse_id in rses_dict]
173             if prioritize_order_over_weight and preferred_rses:
174                 rse = (preferred_rses[0]['rse_id'], preferred_rses[0]['staging_area'], preferred_rses[0]['availability_write'])
175             elif preferred_rses:
176                 rse = self.__choose_rse(preferred_rses)
177             else:
178                 rse = self.__choose_rse(rses)
179             result.append(rse)
180             self.__update_quota(rse, size)
181         return result
182 
183     def get_rse_dictionary(self):
184         """
185         Return the current dictionary of potential RSEs stored in the RSE selector
186 
187         :returns:  List of RSE dictionaries
188         """
189         rse_dict = {}
190         for rse in self.rses:
191             rse_dict[rse['rse_id']] = rse
192         return rse_dict
193 
194     def __update_quota(self, rse, size):
195         """
196         Update the internal quota value.
197 
198         :param rse:      RSE tuple to update.
199         :param size:     Size to substract.
200         """
201 
202         for element in self.rses:
203             if element['rse_id'] == rse[0]:
204                 element['quota_left'] -= size
205                 return
206 
207     def __choose_rse(self, rses):
208         """
209         Choose an RSE based on weighting.
210 
211         :param rses:  The rses to be considered for the choose.
212         :return:      The (rse_id, staging_area) tuple of the chosen RSE.
213         """
214 
215         shuffle(rses)
216         pick = uniform(0, sum([rse['weight'] for rse in rses]))
217         weight = 0
218         for rse in rses:
219             weight += rse['weight']
220             if pick <= weight:
221                 return (rse['rse_id'], rse['staging_area'], rse['availability_write'])
222
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lib/rucio/core/rse_selector.py b/lib/rucio/core/rse_selector.py
--- a/lib/rucio/core/rse_selector.py
+++ b/lib/rucio/core/rse_selector.py
@@ -94,7 +94,9 @@
                         local_quota_left = 0
                     else:
                         local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']
+
                     # check global quota
+                    rse['global_quota_left'] = {}
                     all_global_quota_enough = True
                     for rse_expression, limit in global_quota_limit.items():
                         if rse['rse_id'] in limit['resolved_rse_ids']:
@@ -110,6 +112,8 @@
                             if global_quota_left <= 0:
                                 all_global_quota_enough = False
                                 break
+                            else:
+                                rse['global_quota_left'][rse_expression] = global_quota_left
                     if local_quota_left > 0 and all_global_quota_enough:
                         rse['quota_left'] = local_quota_left
                         space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')
@@ -157,11 +161,25 @@
         if len(rses) < count:
             raise RSEOverQuota('There is insufficient space on any of the target RSE\'s to fullfill the operation.')
 
-        # Remove rses which do not have enough quota
+        # Remove rses which do not have enough local quota
         rses = [rse for rse in rses if rse['quota_left'] > size]
         if len(rses) < count:
             raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
 
+        # Remove rses which do not have enough global quota
+        rses_with_enough_quota = []
+        for rse in rses:
+            enough_global_quota = True
+            for rse_expression in rse.get('global_quota_left', []):
+                if rse['global_quota_left'][rse_expression] < size:
+                    enough_global_quota = False
+                    break
+            if enough_global_quota:
+                rses_with_enough_quota.append(rse)
+        rses = rses_with_enough_quota
+        if len(rses) < count:
+            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\'s to fullfill the operation.')
+
         for copy in range(count):
             # Remove rses already in the result set
             rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]
@@ -202,6 +220,8 @@
         for element in self.rses:
             if element['rse_id'] == rse[0]:
                 element['quota_left'] -= size
+                for rse_expression in element.get('global_quota_left', []):
+                    element['global_quota_left'][rse_expression] -= size
                 return
 
     def __choose_rse(self, rses):

{"golden_diff": "diff --git a/lib/rucio/core/rse_selector.py b/lib/rucio/core/rse_selector.py\n--- a/lib/rucio/core/rse_selector.py\n+++ b/lib/rucio/core/rse_selector.py\n@@ -94,7 +94,9 @@\n                         local_quota_left = 0\n                     else:\n                         local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']\n+\n                     # check global quota\n+                    rse['global_quota_left'] = {}\n                     all_global_quota_enough = True\n                     for rse_expression, limit in global_quota_limit.items():\n                         if rse['rse_id'] in limit['resolved_rse_ids']:\n@@ -110,6 +112,8 @@\n                             if global_quota_left <= 0:\n                                 all_global_quota_enough = False\n                                 break\n+                            else:\n+                                rse['global_quota_left'][rse_expression] = global_quota_left\n                     if local_quota_left > 0 and all_global_quota_enough:\n                         rse['quota_left'] = local_quota_left\n                         space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')\n@@ -157,11 +161,25 @@\n         if len(rses) < count:\n             raise RSEOverQuota('There is insufficient space on any of the target RSE\\'s to fullfill the operation.')\n \n-        # Remove rses which do not have enough quota\n+        # Remove rses which do not have enough local quota\n         rses = [rse for rse in rses if rse['quota_left'] > size]\n         if len(rses) < count:\n             raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n \n+        # Remove rses which do not have enough global quota\n+        rses_with_enough_quota = []\n+        for rse in rses:\n+            enough_global_quota = True\n+            for rse_expression in rse.get('global_quota_left', []):\n+                if rse['global_quota_left'][rse_expression] < size:\n+                    enough_global_quota = False\n+                    break\n+            if enough_global_quota:\n+                rses_with_enough_quota.append(rse)\n+        rses = rses_with_enough_quota\n+        if len(rses) < count:\n+            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n+\n         for copy in range(count):\n             # Remove rses already in the result set\n             rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]\n@@ -202,6 +220,8 @@\n         for element in self.rses:\n             if element['rse_id'] == rse[0]:\n                 element['quota_left'] -= size\n+                for rse_expression in element.get('global_quota_left', []):\n+                    element['global_quota_left'][rse_expression] -= size\n                 return\n \n     def __choose_rse(self, rses):\n", "issue": "check global quota in running RSESelector\nMotivation\r\n----------\r\ncheck global quota not only when initializing the RSESelector but also when calling the select_rse method.\r\n\r\n\r\nModification\r\n------------\r\nupdate select_rse/__update_quota method in the RSESelector\r\n\r\n\n", "before_files": [{"content": "# Copyright European Organization for Nuclear Research (CERN)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# You may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Authors:\n# - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017\n# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2019\n# - Robert Illingworth, <illingwo@fnal.gov>, 2019\n# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019\n# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019\n#\n# PY3K COMPATIBLE\n\nfrom random import uniform, shuffle\n\nfrom rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs, InvalidRuleWeight, RSEOverQuota\nfrom rucio.core.account import has_account_attribute, get_usage, get_all_rse_usages_per_account\nfrom rucio.core.account_limit import get_local_account_limit, get_global_account_limits\nfrom rucio.core.rse import list_rse_attributes, has_rse_attribute, get_rse_limits\nfrom rucio.core.rse_counter import get_counter as get_rse_counter\nfrom rucio.db.sqla.session import read_session\n\n\nclass RSESelector():\n    \"\"\"\n    Representation of the RSE selector\n    \"\"\"\n\n    @read_session\n    def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None):\n        \"\"\"\n        Initialize the RSE Selector.\n\n        :param account:               Account owning the rule.\n        :param rses:                  List of rse dictionaries.\n        :param weight:                Weighting to use.\n        :param copies:                Number of copies to create.\n        :param ignore_account_limit:  Flag if the quota should be ignored.\n        :param session:               DB Session in use.\n        :raises:                      InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs\n        \"\"\"\n        self.account = account\n        self.rses = []  # [{'rse_id':, 'weight':, 'staging_area'}]\n        self.copies = copies\n        if weight is not None:\n            for rse in rses:\n                attributes = list_rse_attributes(rse_id=rse['id'], session=session)\n                availability_write = True if rse.get('availability', 7) & 2 else False\n                if weight not in attributes:\n                    continue  # The RSE does not have the required weight set, therefore it is ignored\n                try:\n                    self.rses.append({'rse_id': rse['id'],\n                                      'weight': float(attributes[weight]),\n                                      'mock_rse': attributes.get('mock', False),\n                                      'availability_write': availability_write,\n                                      'staging_area': rse['staging_area']})\n                except ValueError:\n                    raise InvalidRuleWeight('The RSE \\'%s\\' has a non-number specified for the weight \\'%s\\'' % (rse['rse'], weight))\n        else:\n            for rse in rses:\n                mock_rse = has_rse_attribute(rse['id'], 'mock', session=session)\n                availability_write = True if rse.get('availability', 7) & 2 else False\n                self.rses.append({'rse_id': rse['id'],\n                                  'weight': 1,\n                                  'mock_rse': mock_rse,\n                                  'availability_write': availability_write,\n                                  'staging_area': rse['staging_area']})\n\n        if len(self.rses) < self.copies:\n            raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses)))\n\n        rses_with_enough_quota = []\n        if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit:\n            for rse in self.rses:\n                rse['quota_left'] = float('inf')\n                rse['space_left'] = float('inf')\n                rses_with_enough_quota.append(rse)\n        else:\n            global_quota_limit = get_global_account_limits(account=account, session=session)\n            all_rse_usages = {usage['rse_id']: usage['bytes'] for usage in get_all_rse_usages_per_account(account=account, session=session)}\n            for rse in self.rses:\n                if rse['mock_rse']:\n                    rse['quota_left'] = float('inf')\n                    rse['space_left'] = float('inf')\n                    rses_with_enough_quota.append(rse)\n                else:\n                    # check local quota\n                    local_quota_left = None\n                    quota_limit = get_local_account_limit(account=account, rse_id=rse['rse_id'], session=session)\n                    if quota_limit is None:\n                        local_quota_left = 0\n                    else:\n                        local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']\n                    # check global quota\n                    all_global_quota_enough = True\n                    for rse_expression, limit in global_quota_limit.items():\n                        if rse['rse_id'] in limit['resolved_rse_ids']:\n                            quota_limit = limit['limit']\n                            global_quota_left = None\n                            if quota_limit is None:\n                                global_quota_left = 0\n                            else:\n                                rse_expression_usage = 0\n                                for rse_id in limit['resolved_rse_ids']:\n                                    rse_expression_usage += all_rse_usages.get(rse_id, 0)\n                                global_quota_left = quota_limit - rse_expression_usage\n                            if global_quota_left <= 0:\n                                all_global_quota_enough = False\n                                break\n                    if local_quota_left > 0 and all_global_quota_enough:\n                        rse['quota_left'] = local_quota_left\n                        space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')\n                        if space_limit is None or space_limit < 0:\n                            rse['space_left'] = float('inf')\n                        else:\n                            rse['space_left'] = space_limit - get_rse_counter(rse_id=rse['rse_id'], session=session)['bytes']\n                        rses_with_enough_quota.append(rse)\n\n        self.rses = rses_with_enough_quota\n        if len(self.rses) < self.copies:\n            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n\n        # don't consider removing rses based on the total space here - because files already on the RSE are taken into account\n        # it is possible to have no space but still be able to fulfil the rule\n\n    def select_rse(self, size, preferred_rse_ids, copies=0, blacklist=[], prioritize_order_over_weight=False, existing_rse_size=None):\n        \"\"\"\n        Select n RSEs to replicate data to.\n\n        :param size:                         Size of the block being replicated.\n        :param preferred_rse_ids:            Ordered list of preferred rses. (If possible replicate to them)\n        :param copies:                       Select this amount of copies, if 0 use the pre-defined rule value.\n        :param blacklist:                    List of blacklisted rses. (Do not put replicas on these sites)\n        :param prioritze_order_over_weight:  Prioritize the order of the preferred_rse_ids list over the picking done by weight.\n        :existing_rse_size:                  Dictionary of size of files already present at each rse\n        :returns:                            List of (RSE_id, staging_area, availability_write) tuples.\n        :raises:                             InsufficientAccountLimit, InsufficientTargetRSEs\n        \"\"\"\n\n        result = []\n        rses = self.rses\n        count = self.copies if copies == 0 else copies\n\n        # Remove blacklisted rses\n        if blacklist:\n            rses = [rse for rse in self.rses if rse['rse_id'] not in blacklist]\n        if len(rses) < count:\n            raise InsufficientTargetRSEs('There are not enough target RSEs to fulfil the request at this time.')\n\n        # Remove rses which do not have enough space, accounting for the files already at each rse\n        if existing_rse_size is None:\n            existing_rse_size = {}\n        rses = [rse for rse in rses if rse['space_left'] >= size - existing_rse_size.get(rse['rse_id'], 0)]\n        if len(rses) < count:\n            raise RSEOverQuota('There is insufficient space on any of the target RSE\\'s to fullfill the operation.')\n\n        # Remove rses which do not have enough quota\n        rses = [rse for rse in rses if rse['quota_left'] > size]\n        if len(rses) < count:\n            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n\n        for copy in range(count):\n            # Remove rses already in the result set\n            rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]\n            rses_dict = {}\n            for rse in rses:\n                rses_dict[rse['rse_id']] = rse\n            # Prioritize the preffered rses\n            preferred_rses = [rses_dict[rse_id] for rse_id in preferred_rse_ids if rse_id in rses_dict]\n            if prioritize_order_over_weight and preferred_rses:\n                rse = (preferred_rses[0]['rse_id'], preferred_rses[0]['staging_area'], preferred_rses[0]['availability_write'])\n            elif preferred_rses:\n                rse = self.__choose_rse(preferred_rses)\n            else:\n                rse = self.__choose_rse(rses)\n            result.append(rse)\n            self.__update_quota(rse, size)\n        return result\n\n    def get_rse_dictionary(self):\n        \"\"\"\n        Return the current dictionary of potential RSEs stored in the RSE selector\n\n        :returns:  List of RSE dictionaries\n        \"\"\"\n        rse_dict = {}\n        for rse in self.rses:\n            rse_dict[rse['rse_id']] = rse\n        return rse_dict\n\n    def __update_quota(self, rse, size):\n        \"\"\"\n        Update the internal quota value.\n\n        :param rse:      RSE tuple to update.\n        :param size:     Size to substract.\n        \"\"\"\n\n        for element in self.rses:\n            if element['rse_id'] == rse[0]:\n                element['quota_left'] -= size\n                return\n\n    def __choose_rse(self, rses):\n        \"\"\"\n        Choose an RSE based on weighting.\n\n        :param rses:  The rses to be considered for the choose.\n        :return:      The (rse_id, staging_area) tuple of the chosen RSE.\n        \"\"\"\n\n        shuffle(rses)\n        pick = uniform(0, sum([rse['weight'] for rse in rses]))\n        weight = 0\n        for rse in rses:\n            weight += rse['weight']\n            if pick <= weight:\n                return (rse['rse_id'], rse['staging_area'], rse['availability_write'])\n", "path": "lib/rucio/core/rse_selector.py"}], "after_files": [{"content": "# Copyright European Organization for Nuclear Research (CERN)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# You may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Authors:\n# - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017\n# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2019\n# - Robert Illingworth, <illingwo@fnal.gov>, 2019\n# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019\n# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019\n#\n# PY3K COMPATIBLE\n\nfrom random import uniform, shuffle\n\nfrom rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs, InvalidRuleWeight, RSEOverQuota\nfrom rucio.core.account import has_account_attribute, get_usage, get_all_rse_usages_per_account\nfrom rucio.core.account_limit import get_local_account_limit, get_global_account_limits\nfrom rucio.core.rse import list_rse_attributes, has_rse_attribute, get_rse_limits\nfrom rucio.core.rse_counter import get_counter as get_rse_counter\nfrom rucio.db.sqla.session import read_session\n\n\nclass RSESelector():\n    \"\"\"\n    Representation of the RSE selector\n    \"\"\"\n\n    @read_session\n    def __init__(self, account, rses, weight, copies, ignore_account_limit=False, session=None):\n        \"\"\"\n        Initialize the RSE Selector.\n\n        :param account:               Account owning the rule.\n        :param rses:                  List of rse dictionaries.\n        :param weight:                Weighting to use.\n        :param copies:                Number of copies to create.\n        :param ignore_account_limit:  Flag if the quota should be ignored.\n        :param session:               DB Session in use.\n        :raises:                      InvalidRuleWeight, InsufficientAccountLimit, InsufficientTargetRSEs\n        \"\"\"\n        self.account = account\n        self.rses = []  # [{'rse_id':, 'weight':, 'staging_area'}]\n        self.copies = copies\n        if weight is not None:\n            for rse in rses:\n                attributes = list_rse_attributes(rse_id=rse['id'], session=session)\n                availability_write = True if rse.get('availability', 7) & 2 else False\n                if weight not in attributes:\n                    continue  # The RSE does not have the required weight set, therefore it is ignored\n                try:\n                    self.rses.append({'rse_id': rse['id'],\n                                      'weight': float(attributes[weight]),\n                                      'mock_rse': attributes.get('mock', False),\n                                      'availability_write': availability_write,\n                                      'staging_area': rse['staging_area']})\n                except ValueError:\n                    raise InvalidRuleWeight('The RSE \\'%s\\' has a non-number specified for the weight \\'%s\\'' % (rse['rse'], weight))\n        else:\n            for rse in rses:\n                mock_rse = has_rse_attribute(rse['id'], 'mock', session=session)\n                availability_write = True if rse.get('availability', 7) & 2 else False\n                self.rses.append({'rse_id': rse['id'],\n                                  'weight': 1,\n                                  'mock_rse': mock_rse,\n                                  'availability_write': availability_write,\n                                  'staging_area': rse['staging_area']})\n\n        if len(self.rses) < self.copies:\n            raise InsufficientTargetRSEs('Target RSE set not sufficient for number of copies. (%s copies requested, RSE set size %s)' % (self.copies, len(self.rses)))\n\n        rses_with_enough_quota = []\n        if has_account_attribute(account=account, key='admin', session=session) or ignore_account_limit:\n            for rse in self.rses:\n                rse['quota_left'] = float('inf')\n                rse['space_left'] = float('inf')\n                rses_with_enough_quota.append(rse)\n        else:\n            global_quota_limit = get_global_account_limits(account=account, session=session)\n            all_rse_usages = {usage['rse_id']: usage['bytes'] for usage in get_all_rse_usages_per_account(account=account, session=session)}\n            for rse in self.rses:\n                if rse['mock_rse']:\n                    rse['quota_left'] = float('inf')\n                    rse['space_left'] = float('inf')\n                    rses_with_enough_quota.append(rse)\n                else:\n                    # check local quota\n                    local_quota_left = None\n                    quota_limit = get_local_account_limit(account=account, rse_id=rse['rse_id'], session=session)\n                    if quota_limit is None:\n                        local_quota_left = 0\n                    else:\n                        local_quota_left = quota_limit - get_usage(rse_id=rse['rse_id'], account=account, session=session)['bytes']\n\n                    # check global quota\n                    rse['global_quota_left'] = {}\n                    all_global_quota_enough = True\n                    for rse_expression, limit in global_quota_limit.items():\n                        if rse['rse_id'] in limit['resolved_rse_ids']:\n                            quota_limit = limit['limit']\n                            global_quota_left = None\n                            if quota_limit is None:\n                                global_quota_left = 0\n                            else:\n                                rse_expression_usage = 0\n                                for rse_id in limit['resolved_rse_ids']:\n                                    rse_expression_usage += all_rse_usages.get(rse_id, 0)\n                                global_quota_left = quota_limit - rse_expression_usage\n                            if global_quota_left <= 0:\n                                all_global_quota_enough = False\n                                break\n                            else:\n                                rse['global_quota_left'][rse_expression] = global_quota_left\n                    if local_quota_left > 0 and all_global_quota_enough:\n                        rse['quota_left'] = local_quota_left\n                        space_limit = get_rse_limits(name='MaxSpaceAvailable', rse_id=rse['rse_id'], session=session).get('MaxSpaceAvailable')\n                        if space_limit is None or space_limit < 0:\n                            rse['space_left'] = float('inf')\n                        else:\n                            rse['space_left'] = space_limit - get_rse_counter(rse_id=rse['rse_id'], session=session)['bytes']\n                        rses_with_enough_quota.append(rse)\n\n        self.rses = rses_with_enough_quota\n        if len(self.rses) < self.copies:\n            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n\n        # don't consider removing rses based on the total space here - because files already on the RSE are taken into account\n        # it is possible to have no space but still be able to fulfil the rule\n\n    def select_rse(self, size, preferred_rse_ids, copies=0, blacklist=[], prioritize_order_over_weight=False, existing_rse_size=None):\n        \"\"\"\n        Select n RSEs to replicate data to.\n\n        :param size:                         Size of the block being replicated.\n        :param preferred_rse_ids:            Ordered list of preferred rses. (If possible replicate to them)\n        :param copies:                       Select this amount of copies, if 0 use the pre-defined rule value.\n        :param blacklist:                    List of blacklisted rses. (Do not put replicas on these sites)\n        :param prioritze_order_over_weight:  Prioritize the order of the preferred_rse_ids list over the picking done by weight.\n        :existing_rse_size:                  Dictionary of size of files already present at each rse\n        :returns:                            List of (RSE_id, staging_area, availability_write) tuples.\n        :raises:                             InsufficientAccountLimit, InsufficientTargetRSEs\n        \"\"\"\n\n        result = []\n        rses = self.rses\n        count = self.copies if copies == 0 else copies\n\n        # Remove blacklisted rses\n        if blacklist:\n            rses = [rse for rse in self.rses if rse['rse_id'] not in blacklist]\n        if len(rses) < count:\n            raise InsufficientTargetRSEs('There are not enough target RSEs to fulfil the request at this time.')\n\n        # Remove rses which do not have enough space, accounting for the files already at each rse\n        if existing_rse_size is None:\n            existing_rse_size = {}\n        rses = [rse for rse in rses if rse['space_left'] >= size - existing_rse_size.get(rse['rse_id'], 0)]\n        if len(rses) < count:\n            raise RSEOverQuota('There is insufficient space on any of the target RSE\\'s to fullfill the operation.')\n\n        # Remove rses which do not have enough local quota\n        rses = [rse for rse in rses if rse['quota_left'] > size]\n        if len(rses) < count:\n            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n\n        # Remove rses which do not have enough global quota\n        rses_with_enough_quota = []\n        for rse in rses:\n            enough_global_quota = True\n            for rse_expression in rse.get('global_quota_left', []):\n                if rse['global_quota_left'][rse_expression] < size:\n                    enough_global_quota = False\n                    break\n            if enough_global_quota:\n                rses_with_enough_quota.append(rse)\n        rses = rses_with_enough_quota\n        if len(rses) < count:\n            raise InsufficientAccountLimit('There is insufficient quota on any of the target RSE\\'s to fullfill the operation.')\n\n        for copy in range(count):\n            # Remove rses already in the result set\n            rses = [rse for rse in rses if rse['rse_id'] not in [item[0] for item in result]]\n            rses_dict = {}\n            for rse in rses:\n                rses_dict[rse['rse_id']] = rse\n            # Prioritize the preffered rses\n            preferred_rses = [rses_dict[rse_id] for rse_id in preferred_rse_ids if rse_id in rses_dict]\n            if prioritize_order_over_weight and preferred_rses:\n                rse = (preferred_rses[0]['rse_id'], preferred_rses[0]['staging_area'], preferred_rses[0]['availability_write'])\n            elif preferred_rses:\n                rse = self.__choose_rse(preferred_rses)\n            else:\n                rse = self.__choose_rse(rses)\n            result.append(rse)\n            self.__update_quota(rse, size)\n        return result\n\n    def get_rse_dictionary(self):\n        \"\"\"\n        Return the current dictionary of potential RSEs stored in the RSE selector\n\n        :returns:  List of RSE dictionaries\n        \"\"\"\n        rse_dict = {}\n        for rse in self.rses:\n            rse_dict[rse['rse_id']] = rse\n        return rse_dict\n\n    def __update_quota(self, rse, size):\n        \"\"\"\n        Update the internal quota value.\n\n        :param rse:      RSE tuple to update.\n        :param size:     Size to substract.\n        \"\"\"\n\n        for element in self.rses:\n            if element['rse_id'] == rse[0]:\n                element['quota_left'] -= size\n                for rse_expression in element.get('global_quota_left', []):\n                    element['global_quota_left'][rse_expression] -= size\n                return\n\n    def __choose_rse(self, rses):\n        \"\"\"\n        Choose an RSE based on weighting.\n\n        :param rses:  The rses to be considered for the choose.\n        :return:      The (rse_id, staging_area) tuple of the chosen RSE.\n        \"\"\"\n\n        shuffle(rses)\n        pick = uniform(0, sum([rse['weight'] for rse in rses]))\n        weight = 0\n        for rse in rses:\n            weight += rse['weight']\n            if pick <= weight:\n                return (rse['rse_id'], rse['staging_area'], rse['availability_write'])\n", "path": "lib/rucio/core/rse_selector.py"}]}