<jupyter_start><jupyter_text>Extracting tests from notebooks> The functions that grab the cells containing tests (filtering with potential flags) and execute them<jupyter_code># export
_re_all_flag = re.compile("""
# Matches any line with #all_something and catches that something in a group:
^         # beginning of line (since re.MULTILINE is passed)
\s*       # any number of whitespace
\#\s*     # # then any number of whitespace
all_(\S+) # all_ followed by a group with any non-whitespace chars
\s*       # any number of whitespace
$         # end of line (since re.MULTILINE is passed)
""", re.IGNORECASE | re.MULTILINE | re.VERBOSE)
# export
def check_all_flag(cells):
    for cell in cells:
        if check_re(cell, _re_all_flag): return check_re(cell, _re_all_flag).groups()[0]
nb = read_nb("35_tutorial_wikitext.ipynb")
test_eq(check_all_flag(nb['cells']), 'slow')
nb = read_nb("91_notebook_export.ipynb")
assert check_all_flag(nb['cells']) is None
# export
_re_flags = re.compile("""
# Matches any line with a test flad and catches it in a group:
^               # beginning of line (since re.MULTILINE is passed)
\s*             # any number of whitespace
\#\s*           # # then any number of whitespace
(slow|cuda|cpp) # all test flags
\s*             # any number of whitespace
$               # end of line (since re.MULTILINE is passed)
""", re.IGNORECASE | re.MULTILINE | re.VERBOSE)
# export
def get_cell_flags(cell):
    if cell['cell_type'] != 'code': return []
    return _re_flags.findall(cell['source'])
test_eq(get_cell_flags({'cell_type': 'code', 'source': "#hide\n# slow\n"}), ['slow'])
test_eq(get_cell_flags({'cell_type': 'code', 'source': "#hide\n# slow\n # cuda"}), ['slow', 'cuda'])
test_eq(get_cell_flags({'cell_type': 'markdown', 'source': "#hide\n# slow\n # cuda"}), [])
test_eq(get_cell_flags({'cell_type': 'code', 'source': "#hide\n"}), [])
# export
def _add_import_cell(mod):
    "Return an import cell for `mod`"
    return {'cell_type': 'code',
            'execution_count': None,
            'metadata': {'hide_input': True},
            'outputs': [],
            'source': f"\nfrom local.{mod} import *"}
# export
_re_is_export = re.compile(r"""
# Matches any text with #export or #exports flag:
^         # beginning of line (since re.MULTILINE is passed)
\s*       # any number of whitespace
\#\s*     # # then any number of whitespace
exports?  # export or exports
\s*       # any number of whitespace
""", re.IGNORECASE | re.MULTILINE | re.VERBOSE)
# export
_re_has_import = re.compile(r"""
# Matches any text with import statement:
^         # beginning of line (since re.MULTILINE is passed)
\s*       # any number of whitespace
import    # # then any number of whitespace
\s+  
|
\s*
from
\s+\S+\s+
import
\s+
""", re.IGNORECASE | re.MULTILINE | re.VERBOSE)
# export
class NoExportPreprocessor(ExecutePreprocessor):
    "An `ExecutePreprocessor` that executes not exported cells"
    @delegates(ExecutePreprocessor.__init__)
    def __init__(self, flags, **kwargs):
        self.flags = flags
        super().__init__(**kwargs)
        
    def preprocess_cell(self, cell, resources, index):
        if 'source' not in cell or cell['cell_type'] != "code": return cell, resources
        #if _re_is_export.search(cell['source']) and not _re_has_import.search(cell['source']): 
        #    return cell, resources
        for f in get_cell_flags(cell):
            if f not in self.flags:  return cell, resources
        res = super().preprocess_cell(cell, resources, index)
        return res
# export
def test_nb(fn, flags=None):
    "Execute `nb` (or only the `show_doc` cells) with `metadata`"
    os.environ["IN_TEST"] = '1'
    try:
        nb = read_nb(fn)
        all_flag = check_all_flag(nb['cells'])
        if all_flag is not None and all_flag not in L(flags): return
        mod = find_default_export(nb['cells'])
        #if mod is not None: nb['cells'].insert(0, _add_import_cell(mod))
        ep = NoExportPreprocessor(L(flags), timeout=600, kernel_name='python3')
        pnb = nbformat.from_dict(nb)
        ep.preprocess(pnb)
    except Exception as e: 
        print(f"Error in {fn}")
        raise e
    finally: os.environ.pop("IN_TEST")
test_nb("07_vision_core.ipynb")<jupyter_output><empty_output><jupyter_text>Export-<jupyter_code>#hide
notebook2script(all_fs=True)<jupyter_output>Converted 00_test.ipynb.
Converted 01_core.ipynb.
Converted 01a_dispatch.ipynb.
Converted 01b_torch_core.ipynb.
Converted 02_script.ipynb.
Converted 03_dataloader.ipynb.
Converted 04_transform.ipynb.
Converted 05_data_core.ipynb.
Converted 06_data_transforms.ipynb.
Converted 07_vision_core.ipynb.
Converted 08_pets_tutorial.ipynb.
Converted 09_vision_augment.ipynb.
Converted 10_data_block.ipynb.
Converted 11_layers.ipynb.
Converted 11a_vision_models_xresnet.ipynb.
Converted 12_optimizer.ipynb.
Converted 13_learner.ipynb.
Converted 14_callback_schedule.ipynb.
Converted 14a_callback_data.ipynb.
Converted 15_callback_hook.ipynb.
Converted 16_callback_progress.ipynb.
Converted 17_callback_tracker.ipynb.
Converted 18_callback_fp16.ipynb.
Converted 19_callback_mixup.ipynb.
Converted 20_metrics.ipynb.
Converted 21_tutorial_imagenette.ipynb.
Converted 22_vision_learner.ipynb.
Converted 23_tutorial_transfer_learning.ipynb.
Converted 30_text_core.ipynb.
Converted 31_text_data.ipynb.
Converted 32_[...]