We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Honor reusable.top_level: false for generated ES templates
When setting `reusable.top_level: false` the generated Elasticsearch template still includes the field in the template at the top level. I would not expect this field to be included at the top level of the template when this is `false`.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/generator.py`
Content:
```
1 import argparse
2 import glob
3 import os
4 import schema_reader
5 import yaml
6 from generators import intermediate_files
7 from generators import csv_generator
8 from generators import es_template
9 from generators import beats
10 from generators import asciidoc_fields
11 from generators import ecs_helpers
12 
13 
14 def main():
15     args = argument_parser()
16 
17     ecs_version = read_version()
18     print('Running generator. ECS version ' + ecs_version)
19 
20     # Load the default schemas
21     print('Loading default schemas')
22     intermediate_fields = schema_reader.load_schemas()
23 
24     # Maybe load user specified directory of schemas
25     if args.include:
26         include_glob = ecs_helpers.get_glob_files(args.include, ecs_helpers.YAML_EXT)
27 
28         print('Loading user defined schemas: {0}'.format(include_glob))
29 
30         intermediate_custom = schema_reader.load_schemas(include_glob)
31         schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom)
32 
33     schema_reader.assemble_reusables(intermediate_fields)
34 
35     if args.subset:
36         subset = {}
37         for arg in args.subset:
38             for file in glob.glob(arg):
39                 with open(file) as f:
40                     raw = yaml.safe_load(f.read())
41                     ecs_helpers.recursive_merge_subset_dicts(subset, raw)
42         if not subset:
43             raise ValueError('Subset option specified but no subsets found')
44         intermediate_fields = ecs_helpers.fields_subset(subset, intermediate_fields)
45 
46     (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields)
47 
48     # default location to save files
49     out_dir = 'generated'
50     docs_dir = 'docs'
51     if args.out:
52         out_dir = os.path.join(args.out, out_dir)
53         docs_dir = os.path.join(args.out, docs_dir)
54 
55     ecs_helpers.make_dirs(out_dir)
56     ecs_helpers.make_dirs(docs_dir)
57 
58     intermediate_files.generate(nested, flat, out_dir)
59     if args.intermediate_only:
60         exit()
61 
62     csv_generator.generate(flat, ecs_version, out_dir)
63     es_template.generate(flat, ecs_version, out_dir)
64     if args.include or args.subset:
65         exit()
66     beats.generate(nested, ecs_version, out_dir)
67     asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)
68 
69 
70 def argument_parser():
71     parser = argparse.ArgumentParser()
72     parser.add_argument('--intermediate-only', action='store_true',
73                         help='generate intermediary files only')
74     parser.add_argument('--include', nargs='+',
75                         help='include user specified directory of custom field definitions')
76     parser.add_argument('--subset', nargs='+',
77                         help='render a subset of the schema')
78     parser.add_argument('--out', action='store', help='directory to store the generated files')
79     return parser.parse_args()
80 
81 
82 def read_version(file='version'):
83     with open(file, 'r') as infile:
84         return infile.read().rstrip()
85 
86 
87 if __name__ == '__main__':
88     main()
89
```

Path: `scripts/schema_reader.py`
Content:
```
1 import os
2 import yaml
3 import copy
4 from generators import ecs_helpers
5 
6 # This script has a few entrypoints. The code related to each entrypoint is grouped
7 # together between comments.
8 #
9 # load_schemas()
10 #   yml file load (ECS or custom) + cleanup of field set attributes.
11 # merge_schema_fields()
12 #   Merge ECS field sets with custom field sets
13 # generate_nested_flat()
14 #   Finalize the intermediate representation of all fields. Fills field defaults,
15 #   performs field nestings, and precalculates many values used by various generators.
16 
17 # Loads schemas and perform cleanup of schema attributes
18 
19 
20 def load_schemas(files=ecs_helpers.ecs_files()):
21     """Loads the list of files and performs schema level cleanup"""
22     fields_intermediate = load_schema_files(files)
23     finalize_schemas(fields_intermediate)
24     return fields_intermediate
25 
26 
27 def load_schema_files(files):
28     fields_nested = {}
29     for f in files:
30         new_fields = read_schema_file(f)
31         fields_nested.update(new_fields)
32     return fields_nested
33 
34 
35 def read_schema_file(file):
36     """Read a raw schema yml into a map, removing the wrapping array in each file"""
37     with open(file) as f:
38         raw = yaml.safe_load(f.read())
39     fields = {}
40     for field_set in raw:
41         fields[field_set['name']] = field_set
42     return fields
43 
44 
45 def finalize_schemas(fields_nested):
46     """Clean up all schema level attributes"""
47     for schema_name in fields_nested:
48         schema = fields_nested[schema_name]
49         schema_cleanup_values(schema)
50 
51 
52 def schema_cleanup_values(schema):
53     """Clean up one schema"""
54     ecs_helpers.dict_clean_string_values(schema)
55     schema_set_default_values(schema)
56     schema_set_fieldset_prefix(schema)
57     schema_fields_as_dictionary(schema)
58 
59 
60 def schema_set_default_values(schema):
61     schema['type'] = 'group'
62     schema.setdefault('group', 2)
63     schema.setdefault('short', schema['description'])
64     if "\n" in schema['short']:
65         raise ValueError("Short descriptions must be single line.\nFieldset: {}\n{}".format(schema['name'], schema))
66 
67 
68 def schema_set_fieldset_prefix(schema):
69     if 'root' in schema and schema['root']:
70         schema['prefix'] = ''
71     else:
72         schema['prefix'] = schema['name'] + '.'
73 
74 
75 def schema_fields_as_dictionary(schema):
76     """Re-nest the array of field names as a dictionary of 'fieldname' => { field definition }"""
77     field_array = schema.pop('fields', [])
78     schema['fields'] = {}
79     for field in field_array:
80         nested_levels = field['name'].split('.')
81         nested_schema = schema['fields']
82         for level in nested_levels[:-1]:
83             if level not in nested_schema:
84                 nested_schema[level] = {}
85             if 'fields' not in nested_schema[level]:
86                 nested_schema[level]['fields'] = {}
87             nested_schema = nested_schema[level]['fields']
88         if nested_levels[-1] not in nested_schema:
89             nested_schema[nested_levels[-1]] = {}
90         # Only leaf fields will have field details so we can identify them later
91         nested_schema[nested_levels[-1]]['field_details'] = field
92 
93 # Merge ECS field sets with custom field sets
94 
95 
96 def merge_schema_fields(a, b):
97     """Merge ECS field sets with custom field sets"""
98     for key in b:
99         if key not in a:
100             a[key] = b[key]
101         else:
102             a_type = a[key].get('field_details', {}).get('type', 'object')
103             b_type = b[key].get('field_details', {}).get('type', 'object')
104             if a_type != b_type:
105                 raise ValueError('Schemas unmergeable: type {} does not match type {}'.format(a_type, b_type))
106             elif a_type not in ['object', 'nested']:
107                 print('Warning: dropping field {}, already defined'.format(key))
108                 continue
109             # reusable should only be found at the top level of a fieldset
110             if 'reusable' in b[key]:
111                 a[key].setdefault('reusable', {})
112                 a[key]['reusable']['top_level'] = a[key]['reusable'].get(
113                     'top_level', False) or b[key]['reusable']['top_level']
114                 a[key]['reusable'].setdefault('expected', [])
115                 a[key]['reusable']['expected'].extend(b[key]['reusable']['expected'])
116             if 'fields' in b[key]:
117                 a[key].setdefault('fields', {})
118                 merge_schema_fields(a[key]['fields'], b[key]['fields'])
119 
120 # Finalize the intermediate representation of all fields.
121 
122 
123 def generate_nested_flat(fields_intermediate):
124     for field_name, field in fields_intermediate.items():
125         nestings = find_nestings(field['fields'], field_name + ".")
126         nestings.sort()
127         if len(nestings) > 0:
128             field['nestings'] = nestings
129     fields_nested = generate_partially_flattened_fields(fields_intermediate)
130     fields_flat = generate_fully_flattened_fields(fields_intermediate)
131     return (fields_nested, fields_flat)
132 
133 
134 def assemble_reusables(fields_nested):
135     # This happens as a second pass, so that all fieldsets have their
136     # fields array replaced with a fields dictionary.
137     for schema_name in fields_nested:
138         schema = fields_nested[schema_name]
139         duplicate_reusable_fieldsets(schema, fields_nested)
140     cleanup_fields_recursive(fields_nested, "")
141 
142 
143 def duplicate_reusable_fieldsets(schema, fields_nested):
144     """Copies reusable field definitions to their expected places"""
145     # Note: across this schema reader, functions are modifying dictionaries passed
146     # as arguments, which is usually a risk of unintended side effects.
147     # Here it simplifies the nesting of 'group' under 'user',
148     # which is in turn reusable in a few places.
149     if 'reusable' in schema:
150         for new_nesting in schema['reusable']['expected']:
151             split_flat_name = new_nesting.split('.')
152             top_level = split_flat_name[0]
153             # List field set names expected under another field set.
154             # E.g. host.nestings = [ 'geo', 'os', 'user' ]
155             nested_schema = fields_nested[top_level]['fields']
156             for level in split_flat_name[1:]:
157                 nested_schema = nested_schema.get(level, None)
158                 if not nested_schema:
159                     raise ValueError('Field {} in path {} not found in schema'.format(level, new_nesting))
160                 if nested_schema.get('reusable', None):
161                     raise ValueError(
162                         'Reusable fields cannot be put inside other reusable fields except when the destination reusable is at the top level')
163                 nested_schema = nested_schema.setdefault('fields', {})
164             nested_schema[schema['name']] = schema
165 
166 
167 def cleanup_fields_recursive(fields, prefix, original_fieldset=None):
168     for (name, field) in fields.items():
169         # Copy field here so reusable field sets become unique copies instead of references to the original set
170         field = field.copy()
171         fields[name] = field
172         temp_original_fieldset = name if ('reusable' in field and prefix != "") else original_fieldset
173         if 'field_details' in field:
174             # Deep copy the field details so we can insert different flat names for each reusable fieldset
175             field_details = copy.deepcopy(field['field_details'])
176             new_flat_name = prefix + name
177             field_details['flat_name'] = new_flat_name
178             field_details['dashed_name'] = new_flat_name.replace('.', '-').replace('_', '-')
179             if temp_original_fieldset:
180                 field_details['original_fieldset'] = temp_original_fieldset
181             ecs_helpers.dict_clean_string_values(field_details)
182             field_set_defaults(field_details)
183             field['field_details'] = field_details
184         if 'fields' in field:
185             field['fields'] = field['fields'].copy()
186             new_prefix = prefix + name + "."
187             if 'root' in field and field['root']:
188                 new_prefix = ""
189             cleanup_fields_recursive(field['fields'], new_prefix, temp_original_fieldset)
190 
191 
192 def field_set_defaults(field):
193     field.setdefault('normalize', [])
194     if field['type'] == 'keyword':
195         field.setdefault('ignore_above', 1024)
196     if field['type'] == 'text':
197         field.setdefault('norms', False)
198     if field['type'] == 'object':
199         field.setdefault('object_type', 'keyword')
200 
201     field.setdefault('short', field['description'])
202     if "\n" in field['short']:
203         raise ValueError("Short descriptions must be single line.\nField: {}\n{}".format(field['flat_name'], field))
204         # print("  Short descriptions must be single line. Field: {}".format(field['flat_name']))
205 
206     if 'index' in field and not field['index']:
207         field.setdefault('doc_values', False)
208     if 'multi_fields' in field:
209         field_set_multi_field_defaults(field)
210 
211 
212 def field_set_multi_field_defaults(parent_field):
213     """Sets defaults for each nested field in the multi_fields array"""
214     for mf in parent_field['multi_fields']:
215         mf.setdefault('name', mf['type'])
216         if mf['type'] == 'text':
217             mf.setdefault('norms', False)
218         mf['flat_name'] = parent_field['flat_name'] + '.' + mf['name']
219 
220 
221 def find_nestings(fields, prefix):
222     """Recursively finds all reusable fields in the fields dictionary."""
223     nestings = []
224     for field_name, field in fields.items():
225         if 'reusable' in field:
226             nestings.append(prefix + field_name)
227         if 'fields' in field:
228             nestings.extend(find_nestings(field['fields'], prefix + field_name + '.'))
229     return nestings
230 
231 
232 def generate_partially_flattened_fields(fields_nested):
233     flat_fields = {}
234     for (name, field) in fields_nested.items():
235         # assigning field.copy() adds all the top level schema fields, has to be a copy since we're about
236         # to reassign the 'fields' key and we don't want to modify fields_nested
237         flat_fields[name] = field.copy()
238         flat_fields[name]['fields'] = flatten_fields(field['fields'], "")
239     return flat_fields
240 
241 
242 def generate_fully_flattened_fields(fields_nested):
243     return flatten_fields(fields_nested, "")
244 
245 
246 def flatten_fields(fields, key_prefix):
247     flat_fields = {}
248     for (name, field) in fields.items():
249         new_key = key_prefix + name
250         if 'field_details' in field:
251             flat_fields[new_key] = field['field_details'].copy()
252         if 'fields' in field:
253             new_prefix = new_key + "."
254             if 'root' in field and field['root']:
255                 new_prefix = ""
256             flat_fields.update(flatten_fields(field['fields'], new_prefix))
257     return flat_fields
258
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/generator.py b/scripts/generator.py
--- a/scripts/generator.py
+++ b/scripts/generator.py
@@ -13,6 +13,9 @@
 
 def main():
     args = argument_parser()
+    # Get rid of empty include
+    if args.include and [''] == args.include:
+        args.include.clear()
 
     ecs_version = read_version()
     print('Running generator. ECS version ' + ecs_version)
diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py
--- a/scripts/schema_reader.py
+++ b/scripts/schema_reader.py
@@ -240,7 +240,16 @@
 
 
 def generate_fully_flattened_fields(fields_nested):
-    return flatten_fields(fields_nested, "")
+    flattened = flatten_fields(remove_non_root_reusables(fields_nested), "")
+    return flattened
+
+
+def remove_non_root_reusables(fields_nested):
+    fields = {}
+    for (name, field) in fields_nested.items():
+        if 'reusable' not in field or ('reusable' in field and field['reusable']['top_level']):
+            fields[name] = field
+    return fields
 
 
 def flatten_fields(fields, key_prefix):

{"golden_diff": "diff --git a/scripts/generator.py b/scripts/generator.py\n--- a/scripts/generator.py\n+++ b/scripts/generator.py\n@@ -13,6 +13,9 @@\n \n def main():\n     args = argument_parser()\n+    # Get rid of empty include\n+    if args.include and [''] == args.include:\n+        args.include.clear()\n \n     ecs_version = read_version()\n     print('Running generator. ECS version ' + ecs_version)\ndiff --git a/scripts/schema_reader.py b/scripts/schema_reader.py\n--- a/scripts/schema_reader.py\n+++ b/scripts/schema_reader.py\n@@ -240,7 +240,16 @@\n \n \n def generate_fully_flattened_fields(fields_nested):\n-    return flatten_fields(fields_nested, \"\")\n+    flattened = flatten_fields(remove_non_root_reusables(fields_nested), \"\")\n+    return flattened\n+\n+\n+def remove_non_root_reusables(fields_nested):\n+    fields = {}\n+    for (name, field) in fields_nested.items():\n+        if 'reusable' not in field or ('reusable' in field and field['reusable']['top_level']):\n+            fields[name] = field\n+    return fields\n \n \n def flatten_fields(fields, key_prefix):\n", "issue": "Honor reusable.top_level: false for generated ES templates\nWhen setting `reusable.top_level: false` the generated Elasticsearch template still includes the field in the template at the top level. I would not expect this field to be included at the top level of the template when this is `false`.\n", "before_files": [{"content": "import argparse\nimport glob\nimport os\nimport schema_reader\nimport yaml\nfrom generators import intermediate_files\nfrom generators import csv_generator\nfrom generators import es_template\nfrom generators import beats\nfrom generators import asciidoc_fields\nfrom generators import ecs_helpers\n\n\ndef main():\n    args = argument_parser()\n\n    ecs_version = read_version()\n    print('Running generator. ECS version ' + ecs_version)\n\n    # Load the default schemas\n    print('Loading default schemas')\n    intermediate_fields = schema_reader.load_schemas()\n\n    # Maybe load user specified directory of schemas\n    if args.include:\n        include_glob = ecs_helpers.get_glob_files(args.include, ecs_helpers.YAML_EXT)\n\n        print('Loading user defined schemas: {0}'.format(include_glob))\n\n        intermediate_custom = schema_reader.load_schemas(include_glob)\n        schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom)\n\n    schema_reader.assemble_reusables(intermediate_fields)\n\n    if args.subset:\n        subset = {}\n        for arg in args.subset:\n            for file in glob.glob(arg):\n                with open(file) as f:\n                    raw = yaml.safe_load(f.read())\n                    ecs_helpers.recursive_merge_subset_dicts(subset, raw)\n        if not subset:\n            raise ValueError('Subset option specified but no subsets found')\n        intermediate_fields = ecs_helpers.fields_subset(subset, intermediate_fields)\n\n    (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields)\n\n    # default location to save files\n    out_dir = 'generated'\n    docs_dir = 'docs'\n    if args.out:\n        out_dir = os.path.join(args.out, out_dir)\n        docs_dir = os.path.join(args.out, docs_dir)\n\n    ecs_helpers.make_dirs(out_dir)\n    ecs_helpers.make_dirs(docs_dir)\n\n    intermediate_files.generate(nested, flat, out_dir)\n    if args.intermediate_only:\n        exit()\n\n    csv_generator.generate(flat, ecs_version, out_dir)\n    es_template.generate(flat, ecs_version, out_dir)\n    if args.include or args.subset:\n        exit()\n    beats.generate(nested, ecs_version, out_dir)\n    asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)\n\n\ndef argument_parser():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--intermediate-only', action='store_true',\n                        help='generate intermediary files only')\n    parser.add_argument('--include', nargs='+',\n                        help='include user specified directory of custom field definitions')\n    parser.add_argument('--subset', nargs='+',\n                        help='render a subset of the schema')\n    parser.add_argument('--out', action='store', help='directory to store the generated files')\n    return parser.parse_args()\n\n\ndef read_version(file='version'):\n    with open(file, 'r') as infile:\n        return infile.read().rstrip()\n\n\nif __name__ == '__main__':\n    main()\n", "path": "scripts/generator.py"}, {"content": "import os\nimport yaml\nimport copy\nfrom generators import ecs_helpers\n\n# This script has a few entrypoints. The code related to each entrypoint is grouped\n# together between comments.\n#\n# load_schemas()\n#   yml file load (ECS or custom) + cleanup of field set attributes.\n# merge_schema_fields()\n#   Merge ECS field sets with custom field sets\n# generate_nested_flat()\n#   Finalize the intermediate representation of all fields. Fills field defaults,\n#   performs field nestings, and precalculates many values used by various generators.\n\n# Loads schemas and perform cleanup of schema attributes\n\n\ndef load_schemas(files=ecs_helpers.ecs_files()):\n    \"\"\"Loads the list of files and performs schema level cleanup\"\"\"\n    fields_intermediate = load_schema_files(files)\n    finalize_schemas(fields_intermediate)\n    return fields_intermediate\n\n\ndef load_schema_files(files):\n    fields_nested = {}\n    for f in files:\n        new_fields = read_schema_file(f)\n        fields_nested.update(new_fields)\n    return fields_nested\n\n\ndef read_schema_file(file):\n    \"\"\"Read a raw schema yml into a map, removing the wrapping array in each file\"\"\"\n    with open(file) as f:\n        raw = yaml.safe_load(f.read())\n    fields = {}\n    for field_set in raw:\n        fields[field_set['name']] = field_set\n    return fields\n\n\ndef finalize_schemas(fields_nested):\n    \"\"\"Clean up all schema level attributes\"\"\"\n    for schema_name in fields_nested:\n        schema = fields_nested[schema_name]\n        schema_cleanup_values(schema)\n\n\ndef schema_cleanup_values(schema):\n    \"\"\"Clean up one schema\"\"\"\n    ecs_helpers.dict_clean_string_values(schema)\n    schema_set_default_values(schema)\n    schema_set_fieldset_prefix(schema)\n    schema_fields_as_dictionary(schema)\n\n\ndef schema_set_default_values(schema):\n    schema['type'] = 'group'\n    schema.setdefault('group', 2)\n    schema.setdefault('short', schema['description'])\n    if \"\\n\" in schema['short']:\n        raise ValueError(\"Short descriptions must be single line.\\nFieldset: {}\\n{}\".format(schema['name'], schema))\n\n\ndef schema_set_fieldset_prefix(schema):\n    if 'root' in schema and schema['root']:\n        schema['prefix'] = ''\n    else:\n        schema['prefix'] = schema['name'] + '.'\n\n\ndef schema_fields_as_dictionary(schema):\n    \"\"\"Re-nest the array of field names as a dictionary of 'fieldname' => { field definition }\"\"\"\n    field_array = schema.pop('fields', [])\n    schema['fields'] = {}\n    for field in field_array:\n        nested_levels = field['name'].split('.')\n        nested_schema = schema['fields']\n        for level in nested_levels[:-1]:\n            if level not in nested_schema:\n                nested_schema[level] = {}\n            if 'fields' not in nested_schema[level]:\n                nested_schema[level]['fields'] = {}\n            nested_schema = nested_schema[level]['fields']\n        if nested_levels[-1] not in nested_schema:\n            nested_schema[nested_levels[-1]] = {}\n        # Only leaf fields will have field details so we can identify them later\n        nested_schema[nested_levels[-1]]['field_details'] = field\n\n# Merge ECS field sets with custom field sets\n\n\ndef merge_schema_fields(a, b):\n    \"\"\"Merge ECS field sets with custom field sets\"\"\"\n    for key in b:\n        if key not in a:\n            a[key] = b[key]\n        else:\n            a_type = a[key].get('field_details', {}).get('type', 'object')\n            b_type = b[key].get('field_details', {}).get('type', 'object')\n            if a_type != b_type:\n                raise ValueError('Schemas unmergeable: type {} does not match type {}'.format(a_type, b_type))\n            elif a_type not in ['object', 'nested']:\n                print('Warning: dropping field {}, already defined'.format(key))\n                continue\n            # reusable should only be found at the top level of a fieldset\n            if 'reusable' in b[key]:\n                a[key].setdefault('reusable', {})\n                a[key]['reusable']['top_level'] = a[key]['reusable'].get(\n                    'top_level', False) or b[key]['reusable']['top_level']\n                a[key]['reusable'].setdefault('expected', [])\n                a[key]['reusable']['expected'].extend(b[key]['reusable']['expected'])\n            if 'fields' in b[key]:\n                a[key].setdefault('fields', {})\n                merge_schema_fields(a[key]['fields'], b[key]['fields'])\n\n# Finalize the intermediate representation of all fields.\n\n\ndef generate_nested_flat(fields_intermediate):\n    for field_name, field in fields_intermediate.items():\n        nestings = find_nestings(field['fields'], field_name + \".\")\n        nestings.sort()\n        if len(nestings) > 0:\n            field['nestings'] = nestings\n    fields_nested = generate_partially_flattened_fields(fields_intermediate)\n    fields_flat = generate_fully_flattened_fields(fields_intermediate)\n    return (fields_nested, fields_flat)\n\n\ndef assemble_reusables(fields_nested):\n    # This happens as a second pass, so that all fieldsets have their\n    # fields array replaced with a fields dictionary.\n    for schema_name in fields_nested:\n        schema = fields_nested[schema_name]\n        duplicate_reusable_fieldsets(schema, fields_nested)\n    cleanup_fields_recursive(fields_nested, \"\")\n\n\ndef duplicate_reusable_fieldsets(schema, fields_nested):\n    \"\"\"Copies reusable field definitions to their expected places\"\"\"\n    # Note: across this schema reader, functions are modifying dictionaries passed\n    # as arguments, which is usually a risk of unintended side effects.\n    # Here it simplifies the nesting of 'group' under 'user',\n    # which is in turn reusable in a few places.\n    if 'reusable' in schema:\n        for new_nesting in schema['reusable']['expected']:\n            split_flat_name = new_nesting.split('.')\n            top_level = split_flat_name[0]\n            # List field set names expected under another field set.\n            # E.g. host.nestings = [ 'geo', 'os', 'user' ]\n            nested_schema = fields_nested[top_level]['fields']\n            for level in split_flat_name[1:]:\n                nested_schema = nested_schema.get(level, None)\n                if not nested_schema:\n                    raise ValueError('Field {} in path {} not found in schema'.format(level, new_nesting))\n                if nested_schema.get('reusable', None):\n                    raise ValueError(\n                        'Reusable fields cannot be put inside other reusable fields except when the destination reusable is at the top level')\n                nested_schema = nested_schema.setdefault('fields', {})\n            nested_schema[schema['name']] = schema\n\n\ndef cleanup_fields_recursive(fields, prefix, original_fieldset=None):\n    for (name, field) in fields.items():\n        # Copy field here so reusable field sets become unique copies instead of references to the original set\n        field = field.copy()\n        fields[name] = field\n        temp_original_fieldset = name if ('reusable' in field and prefix != \"\") else original_fieldset\n        if 'field_details' in field:\n            # Deep copy the field details so we can insert different flat names for each reusable fieldset\n            field_details = copy.deepcopy(field['field_details'])\n            new_flat_name = prefix + name\n            field_details['flat_name'] = new_flat_name\n            field_details['dashed_name'] = new_flat_name.replace('.', '-').replace('_', '-')\n            if temp_original_fieldset:\n                field_details['original_fieldset'] = temp_original_fieldset\n            ecs_helpers.dict_clean_string_values(field_details)\n            field_set_defaults(field_details)\n            field['field_details'] = field_details\n        if 'fields' in field:\n            field['fields'] = field['fields'].copy()\n            new_prefix = prefix + name + \".\"\n            if 'root' in field and field['root']:\n                new_prefix = \"\"\n            cleanup_fields_recursive(field['fields'], new_prefix, temp_original_fieldset)\n\n\ndef field_set_defaults(field):\n    field.setdefault('normalize', [])\n    if field['type'] == 'keyword':\n        field.setdefault('ignore_above', 1024)\n    if field['type'] == 'text':\n        field.setdefault('norms', False)\n    if field['type'] == 'object':\n        field.setdefault('object_type', 'keyword')\n\n    field.setdefault('short', field['description'])\n    if \"\\n\" in field['short']:\n        raise ValueError(\"Short descriptions must be single line.\\nField: {}\\n{}\".format(field['flat_name'], field))\n        # print(\"  Short descriptions must be single line. Field: {}\".format(field['flat_name']))\n\n    if 'index' in field and not field['index']:\n        field.setdefault('doc_values', False)\n    if 'multi_fields' in field:\n        field_set_multi_field_defaults(field)\n\n\ndef field_set_multi_field_defaults(parent_field):\n    \"\"\"Sets defaults for each nested field in the multi_fields array\"\"\"\n    for mf in parent_field['multi_fields']:\n        mf.setdefault('name', mf['type'])\n        if mf['type'] == 'text':\n            mf.setdefault('norms', False)\n        mf['flat_name'] = parent_field['flat_name'] + '.' + mf['name']\n\n\ndef find_nestings(fields, prefix):\n    \"\"\"Recursively finds all reusable fields in the fields dictionary.\"\"\"\n    nestings = []\n    for field_name, field in fields.items():\n        if 'reusable' in field:\n            nestings.append(prefix + field_name)\n        if 'fields' in field:\n            nestings.extend(find_nestings(field['fields'], prefix + field_name + '.'))\n    return nestings\n\n\ndef generate_partially_flattened_fields(fields_nested):\n    flat_fields = {}\n    for (name, field) in fields_nested.items():\n        # assigning field.copy() adds all the top level schema fields, has to be a copy since we're about\n        # to reassign the 'fields' key and we don't want to modify fields_nested\n        flat_fields[name] = field.copy()\n        flat_fields[name]['fields'] = flatten_fields(field['fields'], \"\")\n    return flat_fields\n\n\ndef generate_fully_flattened_fields(fields_nested):\n    return flatten_fields(fields_nested, \"\")\n\n\ndef flatten_fields(fields, key_prefix):\n    flat_fields = {}\n    for (name, field) in fields.items():\n        new_key = key_prefix + name\n        if 'field_details' in field:\n            flat_fields[new_key] = field['field_details'].copy()\n        if 'fields' in field:\n            new_prefix = new_key + \".\"\n            if 'root' in field and field['root']:\n                new_prefix = \"\"\n            flat_fields.update(flatten_fields(field['fields'], new_prefix))\n    return flat_fields\n", "path": "scripts/schema_reader.py"}], "after_files": [{"content": "import argparse\nimport glob\nimport os\nimport schema_reader\nimport yaml\nfrom generators import intermediate_files\nfrom generators import csv_generator\nfrom generators import es_template\nfrom generators import beats\nfrom generators import asciidoc_fields\nfrom generators import ecs_helpers\n\n\ndef main():\n    args = argument_parser()\n    # Get rid of empty include\n    if args.include and [''] == args.include:\n        args.include.clear()\n\n    ecs_version = read_version()\n    print('Running generator. ECS version ' + ecs_version)\n\n    # Load the default schemas\n    print('Loading default schemas')\n    intermediate_fields = schema_reader.load_schemas()\n\n    # Maybe load user specified directory of schemas\n    if args.include:\n        include_glob = ecs_helpers.get_glob_files(args.include, ecs_helpers.YAML_EXT)\n\n        print('Loading user defined schemas: {0}'.format(include_glob))\n\n        intermediate_custom = schema_reader.load_schemas(include_glob)\n        schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom)\n\n    schema_reader.assemble_reusables(intermediate_fields)\n\n    if args.subset:\n        subset = {}\n        for arg in args.subset:\n            for file in glob.glob(arg):\n                with open(file) as f:\n                    raw = yaml.safe_load(f.read())\n                    ecs_helpers.recursive_merge_subset_dicts(subset, raw)\n        if not subset:\n            raise ValueError('Subset option specified but no subsets found')\n        intermediate_fields = ecs_helpers.fields_subset(subset, intermediate_fields)\n\n    (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields)\n\n    # default location to save files\n    out_dir = 'generated'\n    docs_dir = 'docs'\n    if args.out:\n        out_dir = os.path.join(args.out, out_dir)\n        docs_dir = os.path.join(args.out, docs_dir)\n\n    ecs_helpers.make_dirs(out_dir)\n    ecs_helpers.make_dirs(docs_dir)\n\n    intermediate_files.generate(nested, flat, out_dir)\n    if args.intermediate_only:\n        exit()\n\n    csv_generator.generate(flat, ecs_version, out_dir)\n    es_template.generate(flat, ecs_version, out_dir)\n    if args.include or args.subset:\n        exit()\n    beats.generate(nested, ecs_version, out_dir)\n    asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)\n\n\ndef argument_parser():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('--intermediate-only', action='store_true',\n                        help='generate intermediary files only')\n    parser.add_argument('--include', nargs='+',\n                        help='include user specified directory of custom field definitions')\n    parser.add_argument('--subset', nargs='+',\n                        help='render a subset of the schema')\n    parser.add_argument('--out', action='store', help='directory to store the generated files')\n    return parser.parse_args()\n\n\ndef read_version(file='version'):\n    with open(file, 'r') as infile:\n        return infile.read().rstrip()\n\n\nif __name__ == '__main__':\n    main()\n", "path": "scripts/generator.py"}, {"content": "import os\nimport yaml\nimport copy\nfrom generators import ecs_helpers\n\n# This script has a few entrypoints. The code related to each entrypoint is grouped\n# together between comments.\n#\n# load_schemas()\n#   yml file load (ECS or custom) + cleanup of field set attributes.\n# merge_schema_fields()\n#   Merge ECS field sets with custom field sets\n# generate_nested_flat()\n#   Finalize the intermediate representation of all fields. Fills field defaults,\n#   performs field nestings, and precalculates many values used by various generators.\n\n# Loads schemas and perform cleanup of schema attributes\n\n\ndef load_schemas(files=ecs_helpers.ecs_files()):\n    \"\"\"Loads the list of files and performs schema level cleanup\"\"\"\n    fields_intermediate = load_schema_files(files)\n    finalize_schemas(fields_intermediate)\n    return fields_intermediate\n\n\ndef load_schema_files(files):\n    fields_nested = {}\n    for f in files:\n        new_fields = read_schema_file(f)\n        fields_nested.update(new_fields)\n    return fields_nested\n\n\ndef read_schema_file(file):\n    \"\"\"Read a raw schema yml into a map, removing the wrapping array in each file\"\"\"\n    with open(file) as f:\n        raw = yaml.safe_load(f.read())\n    fields = {}\n    for field_set in raw:\n        fields[field_set['name']] = field_set\n    return fields\n\n\ndef finalize_schemas(fields_nested):\n    \"\"\"Clean up all schema level attributes\"\"\"\n    for schema_name in fields_nested:\n        schema = fields_nested[schema_name]\n        schema_cleanup_values(schema)\n\n\ndef schema_cleanup_values(schema):\n    \"\"\"Clean up one schema\"\"\"\n    ecs_helpers.dict_clean_string_values(schema)\n    schema_set_default_values(schema)\n    schema_set_fieldset_prefix(schema)\n    schema_fields_as_dictionary(schema)\n\n\ndef schema_set_default_values(schema):\n    schema['type'] = 'group'\n    schema.setdefault('group', 2)\n    schema.setdefault('short', schema['description'])\n    if \"\\n\" in schema['short']:\n        raise ValueError(\"Short descriptions must be single line.\\nFieldset: {}\\n{}\".format(schema['name'], schema))\n\n\ndef schema_set_fieldset_prefix(schema):\n    if 'root' in schema and schema['root']:\n        schema['prefix'] = ''\n    else:\n        schema['prefix'] = schema['name'] + '.'\n\n\ndef schema_fields_as_dictionary(schema):\n    \"\"\"Re-nest the array of field names as a dictionary of 'fieldname' => { field definition }\"\"\"\n    field_array = schema.pop('fields', [])\n    schema['fields'] = {}\n    for field in field_array:\n        nested_levels = field['name'].split('.')\n        nested_schema = schema['fields']\n        for level in nested_levels[:-1]:\n            if level not in nested_schema:\n                nested_schema[level] = {}\n            if 'fields' not in nested_schema[level]:\n                nested_schema[level]['fields'] = {}\n            nested_schema = nested_schema[level]['fields']\n        if nested_levels[-1] not in nested_schema:\n            nested_schema[nested_levels[-1]] = {}\n        # Only leaf fields will have field details so we can identify them later\n        nested_schema[nested_levels[-1]]['field_details'] = field\n\n# Merge ECS field sets with custom field sets\n\n\ndef merge_schema_fields(a, b):\n    \"\"\"Merge ECS field sets with custom field sets\"\"\"\n    for key in b:\n        if key not in a:\n            a[key] = b[key]\n        else:\n            a_type = a[key].get('field_details', {}).get('type', 'object')\n            b_type = b[key].get('field_details', {}).get('type', 'object')\n            if a_type != b_type:\n                raise ValueError('Schemas unmergeable: type {} does not match type {}'.format(a_type, b_type))\n            elif a_type not in ['object', 'nested']:\n                print('Warning: dropping field {}, already defined'.format(key))\n                continue\n            # reusable should only be found at the top level of a fieldset\n            if 'reusable' in b[key]:\n                a[key].setdefault('reusable', {})\n                a[key]['reusable']['top_level'] = a[key]['reusable'].get(\n                    'top_level', False) or b[key]['reusable']['top_level']\n                a[key]['reusable'].setdefault('expected', [])\n                a[key]['reusable']['expected'].extend(b[key]['reusable']['expected'])\n            if 'fields' in b[key]:\n                a[key].setdefault('fields', {})\n                merge_schema_fields(a[key]['fields'], b[key]['fields'])\n\n# Finalize the intermediate representation of all fields.\n\n\ndef generate_nested_flat(fields_intermediate):\n    for field_name, field in fields_intermediate.items():\n        nestings = find_nestings(field['fields'], field_name + \".\")\n        nestings.sort()\n        if len(nestings) > 0:\n            field['nestings'] = nestings\n    fields_nested = generate_partially_flattened_fields(fields_intermediate)\n    fields_flat = generate_fully_flattened_fields(fields_intermediate)\n    return (fields_nested, fields_flat)\n\n\ndef assemble_reusables(fields_nested):\n    # This happens as a second pass, so that all fieldsets have their\n    # fields array replaced with a fields dictionary.\n    for schema_name in fields_nested:\n        schema = fields_nested[schema_name]\n        duplicate_reusable_fieldsets(schema, fields_nested)\n    cleanup_fields_recursive(fields_nested, \"\")\n\n\ndef duplicate_reusable_fieldsets(schema, fields_nested):\n    \"\"\"Copies reusable field definitions to their expected places\"\"\"\n    # Note: across this schema reader, functions are modifying dictionaries passed\n    # as arguments, which is usually a risk of unintended side effects.\n    # Here it simplifies the nesting of 'group' under 'user',\n    # which is in turn reusable in a few places.\n    if 'reusable' in schema:\n        for new_nesting in schema['reusable']['expected']:\n            split_flat_name = new_nesting.split('.')\n            top_level = split_flat_name[0]\n            # List field set names expected under another field set.\n            # E.g. host.nestings = [ 'geo', 'os', 'user' ]\n            nested_schema = fields_nested[top_level]['fields']\n            for level in split_flat_name[1:]:\n                nested_schema = nested_schema.get(level, None)\n                if not nested_schema:\n                    raise ValueError('Field {} in path {} not found in schema'.format(level, new_nesting))\n                if nested_schema.get('reusable', None):\n                    raise ValueError(\n                        'Reusable fields cannot be put inside other reusable fields except when the destination reusable is at the top level')\n                nested_schema = nested_schema.setdefault('fields', {})\n            nested_schema[schema['name']] = schema\n\n\ndef cleanup_fields_recursive(fields, prefix, original_fieldset=None):\n    for (name, field) in fields.items():\n        # Copy field here so reusable field sets become unique copies instead of references to the original set\n        field = field.copy()\n        fields[name] = field\n        temp_original_fieldset = name if ('reusable' in field and prefix != \"\") else original_fieldset\n        if 'field_details' in field:\n            # Deep copy the field details so we can insert different flat names for each reusable fieldset\n            field_details = copy.deepcopy(field['field_details'])\n            new_flat_name = prefix + name\n            field_details['flat_name'] = new_flat_name\n            field_details['dashed_name'] = new_flat_name.replace('.', '-').replace('_', '-')\n            if temp_original_fieldset:\n                field_details['original_fieldset'] = temp_original_fieldset\n            ecs_helpers.dict_clean_string_values(field_details)\n            field_set_defaults(field_details)\n            field['field_details'] = field_details\n        if 'fields' in field:\n            field['fields'] = field['fields'].copy()\n            new_prefix = prefix + name + \".\"\n            if 'root' in field and field['root']:\n                new_prefix = \"\"\n            cleanup_fields_recursive(field['fields'], new_prefix, temp_original_fieldset)\n\n\ndef field_set_defaults(field):\n    field.setdefault('normalize', [])\n    if field['type'] == 'keyword':\n        field.setdefault('ignore_above', 1024)\n    if field['type'] == 'text':\n        field.setdefault('norms', False)\n    if field['type'] == 'object':\n        field.setdefault('object_type', 'keyword')\n\n    field.setdefault('short', field['description'])\n    if \"\\n\" in field['short']:\n        raise ValueError(\"Short descriptions must be single line.\\nField: {}\\n{}\".format(field['flat_name'], field))\n        # print(\"  Short descriptions must be single line. Field: {}\".format(field['flat_name']))\n\n    if 'index' in field and not field['index']:\n        field.setdefault('doc_values', False)\n    if 'multi_fields' in field:\n        field_set_multi_field_defaults(field)\n\n\ndef field_set_multi_field_defaults(parent_field):\n    \"\"\"Sets defaults for each nested field in the multi_fields array\"\"\"\n    for mf in parent_field['multi_fields']:\n        mf.setdefault('name', mf['type'])\n        if mf['type'] == 'text':\n            mf.setdefault('norms', False)\n        mf['flat_name'] = parent_field['flat_name'] + '.' + mf['name']\n\n\ndef find_nestings(fields, prefix):\n    \"\"\"Recursively finds all reusable fields in the fields dictionary.\"\"\"\n    nestings = []\n    for field_name, field in fields.items():\n        if 'reusable' in field:\n            nestings.append(prefix + field_name)\n        if 'fields' in field:\n            nestings.extend(find_nestings(field['fields'], prefix + field_name + '.'))\n    return nestings\n\n\ndef generate_partially_flattened_fields(fields_nested):\n    flat_fields = {}\n    for (name, field) in fields_nested.items():\n        # assigning field.copy() adds all the top level schema fields, has to be a copy since we're about\n        # to reassign the 'fields' key and we don't want to modify fields_nested\n        flat_fields[name] = field.copy()\n        flat_fields[name]['fields'] = flatten_fields(field['fields'], \"\")\n    return flat_fields\n\n\ndef generate_fully_flattened_fields(fields_nested):\n    flattened = flatten_fields(remove_non_root_reusables(fields_nested), \"\")\n    return flattened\n\n\ndef remove_non_root_reusables(fields_nested):\n    fields = {}\n    for (name, field) in fields_nested.items():\n        if 'reusable' not in field or ('reusable' in field and field['reusable']['top_level']):\n            fields[name] = field\n    return fields\n\n\ndef flatten_fields(fields, key_prefix):\n    flat_fields = {}\n    for (name, field) in fields.items():\n        new_key = key_prefix + name\n        if 'field_details' in field:\n            flat_fields[new_key] = field['field_details'].copy()\n        if 'fields' in field:\n            new_prefix = new_key + \".\"\n            if 'root' in field and field['root']:\n                new_prefix = \"\"\n            flat_fields.update(flatten_fields(field['fields'], new_prefix))\n    return flat_fields\n", "path": "scripts/schema_reader.py"}]}