We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Include per-scene metrics in eval.json
It would be useful to see metrics for each scene in addition to metrics averaged over all scenes. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `rastervision/evaluation/classification_evaluation.py`
Content:
```
1 from abc import (ABC, abstractmethod)
2 
3 import json
4 
5 from rastervision.evaluation import ClassEvaluationItem
6 from rastervision.utils.files import str_to_file
7 
8 
9 class ClassificationEvaluation(ABC):
10     """Base class for evaluating predictions for tasks that have classes.
11 
12     Evaluations can be keyed, for instance, if evaluations happen per class.
13     """
14 
15     def __init__(self):
16         self.clear()
17 
18     def clear(self):
19         """Clear the Evaluation."""
20         self.class_to_eval_item = {}
21         self.avg_item = None
22 
23     def set_class_to_eval_item(self, class_to_eval_item):
24         self.class_to_eval_item = class_to_eval_item
25 
26     def get_by_id(self, key):
27         """Gets the evaluation for a particular EvaluationItem key"""
28         return self.class_to_eval_item[key]
29 
30     def has_id(self, key):
31         """Answers whether or not the EvaluationItem key is represented"""
32         return key in self.class_to_eval_item
33 
34     def to_json(self):
35         json_rep = []
36         for eval_item in self.class_to_eval_item.values():
37             json_rep.append(eval_item.to_json())
38         json_rep.append(self.avg_item.to_json())
39         return json_rep
40 
41     def save(self, output_uri):
42         """Save this Evaluation to a file.
43 
44         Args:
45             output_uri: string URI for the file to write.
46         """
47         json_str = json.dumps(self.to_json(), indent=4)
48         str_to_file(json_str, output_uri)
49 
50     def merge(self, evaluation):
51         """Merge Evaluation for another Scene into this one.
52 
53         This is useful for computing the average metrics of a set of scenes.
54         The results of the averaging are stored in this Evaluation.
55 
56         Args:
57             evaluation: Evaluation to merge into this one
58         """
59         if len(self.class_to_eval_item) == 0:
60             self.class_to_eval_item = evaluation.class_to_eval_item
61         else:
62             for key, other_eval_item in \
63                     evaluation.class_to_eval_item.items():
64                 if self.has_id(key):
65                     self.get_by_id(key).merge(other_eval_item)
66                 else:
67                     self.class_to_eval_item[key] = other_eval_item
68 
69         self.compute_avg()
70 
71     def compute_avg(self):
72         """Compute average metrics over all keys."""
73         self.avg_item = ClassEvaluationItem(class_name='average')
74         for eval_item in self.class_to_eval_item.values():
75             self.avg_item.merge(eval_item)
76 
77     @abstractmethod
78     def compute(self, ground_truth_labels, prediction_labels):
79         """Compute metrics for a single scene.
80 
81         Args:
82             ground_truth_labels: Ground Truth labels to evaluate against.
83             prediction_labels: The predicted labels to evaluate.
84         """
85         pass
86
```

Path: `rastervision/evaluation/semantic_segmentation_evaluator.py`
Content:
```
1 import logging
2 
3 from rastervision.data import ActivateMixin
4 from rastervision.rv_config import RVConfig
5 from rastervision.utils.files import (download_if_needed)
6 from rastervision.evaluation import (ClassificationEvaluator,
7                                      SemanticSegmentationEvaluation)
8 
9 log = logging.getLogger(__name__)
10 
11 
12 class SemanticSegmentationEvaluator(ClassificationEvaluator):
13     """Evaluates predictions for a set of scenes.
14     """
15 
16     def __init__(self, class_map, output_uri):
17         super().__init__(class_map, output_uri)
18 
19     def create_evaluation(self):
20         return SemanticSegmentationEvaluation(self.class_map)
21 
22     def process(self, scenes, tmp_dir):
23         evaluation = self.create_evaluation()
24         for scene in scenes:
25             log.info('Computing evaluation for scene {}...'.format(scene.id))
26             label_source = scene.ground_truth_label_source
27             label_store = scene.prediction_label_store
28             with ActivateMixin.compose(label_source, label_store):
29                 ground_truth = label_source.get_labels()
30                 predictions = label_store.get_labels()
31 
32                 if scene.aoi_polygons:
33                     # Filter labels based on AOI.
34                     ground_truth = ground_truth.filter_by_aoi(
35                         scene.aoi_polygons)
36                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
37                 scene_evaluation = self.create_evaluation()
38                 scene_evaluation.compute(ground_truth, predictions)
39                 evaluation.merge(scene_evaluation)
40 
41             if hasattr(label_source, 'source') and hasattr(
42                     label_source.source, 'vector_source') and hasattr(
43                         label_store, 'vector_output'):
44                 tmp_dir = RVConfig.get_tmp_dir().name
45                 gt_geojson = label_source.source.vector_source.get_geojson()
46                 for vo in label_store.vector_output:
47                     pred_geojson = vo['uri']
48                     mode = vo['mode']
49                     class_id = vo['class_id']
50                     pred_geojson_local = download_if_needed(
51                         pred_geojson, tmp_dir)
52                     scene_evaluation = self.create_evaluation()
53                     scene_evaluation.compute_vector(
54                         gt_geojson, pred_geojson_local, mode, class_id)
55                     evaluation.merge(scene_evaluation)
56 
57         evaluation.save(self.output_uri)
58
```

Path: `rastervision/evaluation/classification_evaluator.py`
Content:
```
1 from abc import (abstractmethod)
2 import logging
3 
4 from rastervision.evaluation import Evaluator
5 from rastervision.data import ActivateMixin
6 
7 log = logging.getLogger(__name__)
8 
9 
10 class ClassificationEvaluator(Evaluator):
11     """Evaluates predictions for a set of scenes.
12     """
13 
14     def __init__(self, class_map, output_uri):
15         self.class_map = class_map
16         self.output_uri = output_uri
17 
18     @abstractmethod
19     def create_evaluation(self):
20         pass
21 
22     def process(self, scenes, tmp_dir):
23         evaluation = self.create_evaluation()
24         for scene in scenes:
25             log.info('Computing evaluation for scene {}...'.format(scene.id))
26             label_source = scene.ground_truth_label_source
27             label_store = scene.prediction_label_store
28             with ActivateMixin.compose(label_source, label_store):
29                 ground_truth = label_source.get_labels()
30                 predictions = label_store.get_labels()
31 
32                 if scene.aoi_polygons:
33                     # Filter labels based on AOI.
34                     ground_truth = ground_truth.filter_by_aoi(
35                         scene.aoi_polygons)
36                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
37                 scene_evaluation = self.create_evaluation()
38                 scene_evaluation.compute(ground_truth, predictions)
39                 evaluation.merge(scene_evaluation)
40 
41         evaluation.save(self.output_uri)
42
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/rastervision/evaluation/classification_evaluation.py b/rastervision/evaluation/classification_evaluation.py
--- a/rastervision/evaluation/classification_evaluation.py
+++ b/rastervision/evaluation/classification_evaluation.py
@@ -1,4 +1,5 @@
 from abc import (ABC, abstractmethod)
+import copy
 
 import json
 
@@ -18,6 +19,7 @@
     def clear(self):
         """Clear the Evaluation."""
         self.class_to_eval_item = {}
+        self.scene_to_eval = {}
         self.avg_item = None
 
     def set_class_to_eval_item(self, class_to_eval_item):
@@ -36,6 +38,14 @@
         for eval_item in self.class_to_eval_item.values():
             json_rep.append(eval_item.to_json())
         json_rep.append(self.avg_item.to_json())
+
+        if self.scene_to_eval:
+            json_rep = {'overall': json_rep}
+            scene_to_eval_json = {}
+            for scene_id, eval in self.scene_to_eval.items():
+                scene_to_eval_json[scene_id] = eval.to_json()
+            json_rep['per_scene'] = scene_to_eval_json
+
         return json_rep
 
     def save(self, output_uri):
@@ -47,7 +57,7 @@
         json_str = json.dumps(self.to_json(), indent=4)
         str_to_file(json_str, output_uri)
 
-    def merge(self, evaluation):
+    def merge(self, evaluation, scene_id=None):
         """Merge Evaluation for another Scene into this one.
 
         This is useful for computing the average metrics of a set of scenes.
@@ -68,6 +78,9 @@
 
         self.compute_avg()
 
+        if scene_id is not None:
+            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)
+
     def compute_avg(self):
         """Compute average metrics over all keys."""
         self.avg_item = ClassEvaluationItem(class_name='average')
diff --git a/rastervision/evaluation/classification_evaluator.py b/rastervision/evaluation/classification_evaluator.py
--- a/rastervision/evaluation/classification_evaluator.py
+++ b/rastervision/evaluation/classification_evaluator.py
@@ -36,6 +36,5 @@
                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
                 scene_evaluation = self.create_evaluation()
                 scene_evaluation.compute(ground_truth, predictions)
-                evaluation.merge(scene_evaluation)
-
+                evaluation.merge(scene_evaluation, scene_id=scene.id)
         evaluation.save(self.output_uri)
diff --git a/rastervision/evaluation/semantic_segmentation_evaluator.py b/rastervision/evaluation/semantic_segmentation_evaluator.py
--- a/rastervision/evaluation/semantic_segmentation_evaluator.py
+++ b/rastervision/evaluation/semantic_segmentation_evaluator.py
@@ -36,7 +36,7 @@
                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)
                 scene_evaluation = self.create_evaluation()
                 scene_evaluation.compute(ground_truth, predictions)
-                evaluation.merge(scene_evaluation)
+                evaluation.merge(scene_evaluation, scene_id=scene.id)
 
             if hasattr(label_source, 'source') and hasattr(
                     label_source.source, 'vector_source') and hasattr(
@@ -52,6 +52,6 @@
                     scene_evaluation = self.create_evaluation()
                     scene_evaluation.compute_vector(
                         gt_geojson, pred_geojson_local, mode, class_id)
-                    evaluation.merge(scene_evaluation)
+                    evaluation.merge(scene_evaluation, scene_id=scene.id)
 
         evaluation.save(self.output_uri)

{"golden_diff": "diff --git a/rastervision/evaluation/classification_evaluation.py b/rastervision/evaluation/classification_evaluation.py\n--- a/rastervision/evaluation/classification_evaluation.py\n+++ b/rastervision/evaluation/classification_evaluation.py\n@@ -1,4 +1,5 @@\n from abc import (ABC, abstractmethod)\n+import copy\n \n import json\n \n@@ -18,6 +19,7 @@\n     def clear(self):\n         \"\"\"Clear the Evaluation.\"\"\"\n         self.class_to_eval_item = {}\n+        self.scene_to_eval = {}\n         self.avg_item = None\n \n     def set_class_to_eval_item(self, class_to_eval_item):\n@@ -36,6 +38,14 @@\n         for eval_item in self.class_to_eval_item.values():\n             json_rep.append(eval_item.to_json())\n         json_rep.append(self.avg_item.to_json())\n+\n+        if self.scene_to_eval:\n+            json_rep = {'overall': json_rep}\n+            scene_to_eval_json = {}\n+            for scene_id, eval in self.scene_to_eval.items():\n+                scene_to_eval_json[scene_id] = eval.to_json()\n+            json_rep['per_scene'] = scene_to_eval_json\n+\n         return json_rep\n \n     def save(self, output_uri):\n@@ -47,7 +57,7 @@\n         json_str = json.dumps(self.to_json(), indent=4)\n         str_to_file(json_str, output_uri)\n \n-    def merge(self, evaluation):\n+    def merge(self, evaluation, scene_id=None):\n         \"\"\"Merge Evaluation for another Scene into this one.\n \n         This is useful for computing the average metrics of a set of scenes.\n@@ -68,6 +78,9 @@\n \n         self.compute_avg()\n \n+        if scene_id is not None:\n+            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)\n+\n     def compute_avg(self):\n         \"\"\"Compute average metrics over all keys.\"\"\"\n         self.avg_item = ClassEvaluationItem(class_name='average')\ndiff --git a/rastervision/evaluation/classification_evaluator.py b/rastervision/evaluation/classification_evaluator.py\n--- a/rastervision/evaluation/classification_evaluator.py\n+++ b/rastervision/evaluation/classification_evaluator.py\n@@ -36,6 +36,5 @@\n                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                 scene_evaluation = self.create_evaluation()\n                 scene_evaluation.compute(ground_truth, predictions)\n-                evaluation.merge(scene_evaluation)\n-\n+                evaluation.merge(scene_evaluation, scene_id=scene.id)\n         evaluation.save(self.output_uri)\ndiff --git a/rastervision/evaluation/semantic_segmentation_evaluator.py b/rastervision/evaluation/semantic_segmentation_evaluator.py\n--- a/rastervision/evaluation/semantic_segmentation_evaluator.py\n+++ b/rastervision/evaluation/semantic_segmentation_evaluator.py\n@@ -36,7 +36,7 @@\n                     predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                 scene_evaluation = self.create_evaluation()\n                 scene_evaluation.compute(ground_truth, predictions)\n-                evaluation.merge(scene_evaluation)\n+                evaluation.merge(scene_evaluation, scene_id=scene.id)\n \n             if hasattr(label_source, 'source') and hasattr(\n                     label_source.source, 'vector_source') and hasattr(\n@@ -52,6 +52,6 @@\n                     scene_evaluation = self.create_evaluation()\n                     scene_evaluation.compute_vector(\n                         gt_geojson, pred_geojson_local, mode, class_id)\n-                    evaluation.merge(scene_evaluation)\n+                    evaluation.merge(scene_evaluation, scene_id=scene.id)\n \n         evaluation.save(self.output_uri)\n", "issue": "Include per-scene metrics in eval.json\nIt would be useful to see metrics for each scene in addition to metrics averaged over all scenes. \n", "before_files": [{"content": "from abc import (ABC, abstractmethod)\n\nimport json\n\nfrom rastervision.evaluation import ClassEvaluationItem\nfrom rastervision.utils.files import str_to_file\n\n\nclass ClassificationEvaluation(ABC):\n    \"\"\"Base class for evaluating predictions for tasks that have classes.\n\n    Evaluations can be keyed, for instance, if evaluations happen per class.\n    \"\"\"\n\n    def __init__(self):\n        self.clear()\n\n    def clear(self):\n        \"\"\"Clear the Evaluation.\"\"\"\n        self.class_to_eval_item = {}\n        self.avg_item = None\n\n    def set_class_to_eval_item(self, class_to_eval_item):\n        self.class_to_eval_item = class_to_eval_item\n\n    def get_by_id(self, key):\n        \"\"\"Gets the evaluation for a particular EvaluationItem key\"\"\"\n        return self.class_to_eval_item[key]\n\n    def has_id(self, key):\n        \"\"\"Answers whether or not the EvaluationItem key is represented\"\"\"\n        return key in self.class_to_eval_item\n\n    def to_json(self):\n        json_rep = []\n        for eval_item in self.class_to_eval_item.values():\n            json_rep.append(eval_item.to_json())\n        json_rep.append(self.avg_item.to_json())\n        return json_rep\n\n    def save(self, output_uri):\n        \"\"\"Save this Evaluation to a file.\n\n        Args:\n            output_uri: string URI for the file to write.\n        \"\"\"\n        json_str = json.dumps(self.to_json(), indent=4)\n        str_to_file(json_str, output_uri)\n\n    def merge(self, evaluation):\n        \"\"\"Merge Evaluation for another Scene into this one.\n\n        This is useful for computing the average metrics of a set of scenes.\n        The results of the averaging are stored in this Evaluation.\n\n        Args:\n            evaluation: Evaluation to merge into this one\n        \"\"\"\n        if len(self.class_to_eval_item) == 0:\n            self.class_to_eval_item = evaluation.class_to_eval_item\n        else:\n            for key, other_eval_item in \\\n                    evaluation.class_to_eval_item.items():\n                if self.has_id(key):\n                    self.get_by_id(key).merge(other_eval_item)\n                else:\n                    self.class_to_eval_item[key] = other_eval_item\n\n        self.compute_avg()\n\n    def compute_avg(self):\n        \"\"\"Compute average metrics over all keys.\"\"\"\n        self.avg_item = ClassEvaluationItem(class_name='average')\n        for eval_item in self.class_to_eval_item.values():\n            self.avg_item.merge(eval_item)\n\n    @abstractmethod\n    def compute(self, ground_truth_labels, prediction_labels):\n        \"\"\"Compute metrics for a single scene.\n\n        Args:\n            ground_truth_labels: Ground Truth labels to evaluate against.\n            prediction_labels: The predicted labels to evaluate.\n        \"\"\"\n        pass\n", "path": "rastervision/evaluation/classification_evaluation.py"}, {"content": "import logging\n\nfrom rastervision.data import ActivateMixin\nfrom rastervision.rv_config import RVConfig\nfrom rastervision.utils.files import (download_if_needed)\nfrom rastervision.evaluation import (ClassificationEvaluator,\n                                     SemanticSegmentationEvaluation)\n\nlog = logging.getLogger(__name__)\n\n\nclass SemanticSegmentationEvaluator(ClassificationEvaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        super().__init__(class_map, output_uri)\n\n    def create_evaluation(self):\n        return SemanticSegmentationEvaluation(self.class_map)\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation)\n\n            if hasattr(label_source, 'source') and hasattr(\n                    label_source.source, 'vector_source') and hasattr(\n                        label_store, 'vector_output'):\n                tmp_dir = RVConfig.get_tmp_dir().name\n                gt_geojson = label_source.source.vector_source.get_geojson()\n                for vo in label_store.vector_output:\n                    pred_geojson = vo['uri']\n                    mode = vo['mode']\n                    class_id = vo['class_id']\n                    pred_geojson_local = download_if_needed(\n                        pred_geojson, tmp_dir)\n                    scene_evaluation = self.create_evaluation()\n                    scene_evaluation.compute_vector(\n                        gt_geojson, pred_geojson_local, mode, class_id)\n                    evaluation.merge(scene_evaluation)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/semantic_segmentation_evaluator.py"}, {"content": "from abc import (abstractmethod)\nimport logging\n\nfrom rastervision.evaluation import Evaluator\nfrom rastervision.data import ActivateMixin\n\nlog = logging.getLogger(__name__)\n\n\nclass ClassificationEvaluator(Evaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        self.class_map = class_map\n        self.output_uri = output_uri\n\n    @abstractmethod\n    def create_evaluation(self):\n        pass\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/classification_evaluator.py"}], "after_files": [{"content": "from abc import (ABC, abstractmethod)\nimport copy\n\nimport json\n\nfrom rastervision.evaluation import ClassEvaluationItem\nfrom rastervision.utils.files import str_to_file\n\n\nclass ClassificationEvaluation(ABC):\n    \"\"\"Base class for evaluating predictions for tasks that have classes.\n\n    Evaluations can be keyed, for instance, if evaluations happen per class.\n    \"\"\"\n\n    def __init__(self):\n        self.clear()\n\n    def clear(self):\n        \"\"\"Clear the Evaluation.\"\"\"\n        self.class_to_eval_item = {}\n        self.scene_to_eval = {}\n        self.avg_item = None\n\n    def set_class_to_eval_item(self, class_to_eval_item):\n        self.class_to_eval_item = class_to_eval_item\n\n    def get_by_id(self, key):\n        \"\"\"Gets the evaluation for a particular EvaluationItem key\"\"\"\n        return self.class_to_eval_item[key]\n\n    def has_id(self, key):\n        \"\"\"Answers whether or not the EvaluationItem key is represented\"\"\"\n        return key in self.class_to_eval_item\n\n    def to_json(self):\n        json_rep = []\n        for eval_item in self.class_to_eval_item.values():\n            json_rep.append(eval_item.to_json())\n        json_rep.append(self.avg_item.to_json())\n\n        if self.scene_to_eval:\n            json_rep = {'overall': json_rep}\n            scene_to_eval_json = {}\n            for scene_id, eval in self.scene_to_eval.items():\n                scene_to_eval_json[scene_id] = eval.to_json()\n            json_rep['per_scene'] = scene_to_eval_json\n\n        return json_rep\n\n    def save(self, output_uri):\n        \"\"\"Save this Evaluation to a file.\n\n        Args:\n            output_uri: string URI for the file to write.\n        \"\"\"\n        json_str = json.dumps(self.to_json(), indent=4)\n        str_to_file(json_str, output_uri)\n\n    def merge(self, evaluation, scene_id=None):\n        \"\"\"Merge Evaluation for another Scene into this one.\n\n        This is useful for computing the average metrics of a set of scenes.\n        The results of the averaging are stored in this Evaluation.\n\n        Args:\n            evaluation: Evaluation to merge into this one\n        \"\"\"\n        if len(self.class_to_eval_item) == 0:\n            self.class_to_eval_item = evaluation.class_to_eval_item\n        else:\n            for key, other_eval_item in \\\n                    evaluation.class_to_eval_item.items():\n                if self.has_id(key):\n                    self.get_by_id(key).merge(other_eval_item)\n                else:\n                    self.class_to_eval_item[key] = other_eval_item\n\n        self.compute_avg()\n\n        if scene_id is not None:\n            self.scene_to_eval[scene_id] = copy.deepcopy(evaluation)\n\n    def compute_avg(self):\n        \"\"\"Compute average metrics over all keys.\"\"\"\n        self.avg_item = ClassEvaluationItem(class_name='average')\n        for eval_item in self.class_to_eval_item.values():\n            self.avg_item.merge(eval_item)\n\n    @abstractmethod\n    def compute(self, ground_truth_labels, prediction_labels):\n        \"\"\"Compute metrics for a single scene.\n\n        Args:\n            ground_truth_labels: Ground Truth labels to evaluate against.\n            prediction_labels: The predicted labels to evaluate.\n        \"\"\"\n        pass\n", "path": "rastervision/evaluation/classification_evaluation.py"}, {"content": "import logging\n\nfrom rastervision.data import ActivateMixin\nfrom rastervision.rv_config import RVConfig\nfrom rastervision.utils.files import (download_if_needed)\nfrom rastervision.evaluation import (ClassificationEvaluator,\n                                     SemanticSegmentationEvaluation)\n\nlog = logging.getLogger(__name__)\n\n\nclass SemanticSegmentationEvaluator(ClassificationEvaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        super().__init__(class_map, output_uri)\n\n    def create_evaluation(self):\n        return SemanticSegmentationEvaluation(self.class_map)\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation, scene_id=scene.id)\n\n            if hasattr(label_source, 'source') and hasattr(\n                    label_source.source, 'vector_source') and hasattr(\n                        label_store, 'vector_output'):\n                tmp_dir = RVConfig.get_tmp_dir().name\n                gt_geojson = label_source.source.vector_source.get_geojson()\n                for vo in label_store.vector_output:\n                    pred_geojson = vo['uri']\n                    mode = vo['mode']\n                    class_id = vo['class_id']\n                    pred_geojson_local = download_if_needed(\n                        pred_geojson, tmp_dir)\n                    scene_evaluation = self.create_evaluation()\n                    scene_evaluation.compute_vector(\n                        gt_geojson, pred_geojson_local, mode, class_id)\n                    evaluation.merge(scene_evaluation, scene_id=scene.id)\n\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/semantic_segmentation_evaluator.py"}, {"content": "from abc import (abstractmethod)\nimport logging\n\nfrom rastervision.evaluation import Evaluator\nfrom rastervision.data import ActivateMixin\n\nlog = logging.getLogger(__name__)\n\n\nclass ClassificationEvaluator(Evaluator):\n    \"\"\"Evaluates predictions for a set of scenes.\n    \"\"\"\n\n    def __init__(self, class_map, output_uri):\n        self.class_map = class_map\n        self.output_uri = output_uri\n\n    @abstractmethod\n    def create_evaluation(self):\n        pass\n\n    def process(self, scenes, tmp_dir):\n        evaluation = self.create_evaluation()\n        for scene in scenes:\n            log.info('Computing evaluation for scene {}...'.format(scene.id))\n            label_source = scene.ground_truth_label_source\n            label_store = scene.prediction_label_store\n            with ActivateMixin.compose(label_source, label_store):\n                ground_truth = label_source.get_labels()\n                predictions = label_store.get_labels()\n\n                if scene.aoi_polygons:\n                    # Filter labels based on AOI.\n                    ground_truth = ground_truth.filter_by_aoi(\n                        scene.aoi_polygons)\n                    predictions = predictions.filter_by_aoi(scene.aoi_polygons)\n                scene_evaluation = self.create_evaluation()\n                scene_evaluation.compute(ground_truth, predictions)\n                evaluation.merge(scene_evaluation, scene_id=scene.id)\n        evaluation.save(self.output_uri)\n", "path": "rastervision/evaluation/classification_evaluator.py"}]}