


from manim import *


class   a__	(						lowerCAmelCase_    ):

       def 						UpperCAmelCase(   self      :     List[Any]    ):
            a_    :     Any										=    Rectangle(height=0.5      ,							width=0.5    )
            a_    :     str										=    Rectangle(height=0.4_6      ,							width=0.4_6    ).set_stroke(width=0    )

            a_    :     Optional[Any]										=    Rectangle(height=0.2_5      ,							width=0.2_5    )

            a_    :     Any										=    [mem.copy() for i in range(6    )]
            a_    :     Tuple										=    [mem.copy() for i in range(6    )]
            a_    :     Dict										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     int										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     List[Any]										=    VGroup(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     List[Any]										=    Text("""CPU"""      ,							font_size=2_4    )
            a_    :     str										=    Group(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0.5      ,							aligned_edge=lowerCamelCase_    )
            cpu.move_to([-2.5, -0.5, 0]    )
            self.add(lowerCamelCase_    )

            a_    :     Dict										=    [mem.copy() for i in range(4    )]
            a_    :     Any										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     Optional[int]										=    Text("""GPU"""      ,							font_size=2_4    )
            a_    :     Optional[int]										=    Group(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0.5      ,							aligned_edge=lowerCamelCase_    )
            gpu.move_to([-1, -1, 0]    )
            self.add(lowerCamelCase_    )

            a_    :     List[str]										=    [mem.copy() for i in range(6    )]
            a_    :     str										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )

            a_    :     List[str]										=    Text("""Model"""      ,							font_size=2_4    )
            a_    :     Optional[Any]										=    Group(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0.5      ,							aligned_edge=lowerCamelCase_    )
            model.move_to([3, -1.0, 0]    )
            self.add(lowerCamelCase_    )

            a_    :     Dict										=    []
            a_    :     Tuple										=    []

            for i, rect in enumerate(lowerCamelCase_    ):
                 a_    :     Dict										=    fill.copy().set_fill(lowerCamelCase_      ,							opacity=0.8    )
                 target.move_to(lowerCamelCase_    )
                 model_arr.append(lowerCamelCase_    )

                 a_    :     Optional[int]										=    Rectangle(height=0.4_6      ,							width=0.4_6    ).set_stroke(width=0.0    ).set_fill(lowerCamelCase_      ,							opacity=0.8    )
                 cpu_target.move_to(cpu_left_col_base[i]    )
                 model_cpu_arr.append(lowerCamelCase_    )

            self.add(*lowerCamelCase_      ,							*lowerCamelCase_    )

            a_    :     Dict										=    [meta_mem.copy() for i in range(6    )]
            a_    :     Optional[Any]										=    [meta_mem.copy() for i in range(6    )]
            a_    :     Optional[int]										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     Optional[int]										=    VGroup(*lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     str										=    VGroup(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0    )
            a_    :     Optional[int]										=    Text("""Disk"""      ,							font_size=2_4    )
            a_    :     Any										=    Group(lowerCamelCase_      ,							lowerCamelCase_    ).arrange(lowerCamelCase_      ,							buff=0.5      ,							aligned_edge=lowerCamelCase_    )
            disk.move_to([-4, -1.2_5, 0]    )
            self.add(lowerCamelCase_      ,							lowerCamelCase_    )

            a_    :     Optional[int]										=    Square(side_length=2.2    )
            key.move_to([-5, 2, 0]    )

            a_    :     Any										=    MarkupText(
                F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''      ,							font_size=1_8      ,							)

            key_text.move_to([-5, 2.4, 0]    )

            self.add(lowerCamelCase_      ,							lowerCamelCase_    )

            a_    :     int										=    MarkupText(
                F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''      ,							font_size=1_8      ,							)

            blue_text.next_to(lowerCamelCase_      ,							DOWN * 2.4      ,							aligned_edge=key_text.get_left()    )
            self.add(lowerCamelCase_    )

            a_    :     Optional[int]										=    MarkupText(
                F'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''      ,							font_size=2_4      ,							)
            step_a.move_to([2, 2, 0]    )

            self.play(Write(lowerCamelCase_    )    )

            a_    :     Any										=    Square(0.3    )
            input.set_fill(lowerCamelCase_      ,							opacity=1.0    )
            input.set_stroke(width=0.0    )
            input.next_to(model_base[0]      ,							lowerCamelCase_      ,							buff=0.5    )

            self.play(Write(lowerCamelCase_    )    )

            input.generate_target()
            input.target.next_to(model_arr[0]      ,							direction=lowerCamelCase_      ,							buff=0.0_2    )
            self.play(MoveToTarget(lowerCamelCase_    )    )

            self.play(FadeOut(lowerCamelCase_    )    )

            a_    :     Union[str, Any]										=    Arrow(start=lowerCamelCase_      ,							end=lowerCamelCase_      ,							color=lowerCamelCase_      ,							buff=0.5    )
            a.next_to(model_arr[0].get_left()      ,							lowerCamelCase_      ,							buff=0.2    )

            model_cpu_arr[0].generate_target()
            model_cpu_arr[0].target.move_to(gpu_rect[0]    )

            a_    :     Tuple										=    MarkupText(
                F'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''      ,							font_size=2_4      ,							)
            step_a.move_to([2, 2, 0]    )

            self.play(Write(lowerCamelCase_      ,							run_time=3    )    )

            a_    :     Dict										=    {"""run_time""": 1, """fade_in""": True, """fade_out""": True, """buff""": 0.0_2}

            self.play(
                Write(lowerCamelCase_    )      ,							Circumscribe(model_arr[0]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(model_cpu_arr[0]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(gpu_rect[0]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							)
            self.play(MoveToTarget(model_cpu_arr[0]    )    )

            a_    :     Dict										=    a.copy()
            for i in range(6    ):
                 a_c.next_to(model_arr[i].get_right() + 0.0_2      ,							lowerCamelCase_      ,							buff=0.2    )

                 input.generate_target()
                 input.target.move_to(model_arr[i].get_right() + 0.0_2    )

                 a_    :     str										=    AnimationGroup(
                     FadeOut(lowerCamelCase_      ,							run_time=0.5    )      ,							MoveToTarget(lowerCamelCase_      ,							run_time=0.5    )      ,							FadeIn(lowerCamelCase_      ,							run_time=0.5    )      ,							lag_ratio=0.2    )

                 self.play(lowerCamelCase_    )

                 model_cpu_arr[i].generate_target()
                 model_cpu_arr[i].target.move_to(cpu_left_col_base[i]    )

                 if i < 5:
                      model_cpu_arr[i + 1].generate_target()
                      model_cpu_arr[i + 1].target.move_to(gpu_rect[0]    )
                      if i >= 1:
                           a_    :     List[str]										=    0.7

                      self.play(
                          Circumscribe(model_arr[i]      ,							**lowerCamelCase_    )      ,							Circumscribe(cpu_left_col_base[i]      ,							**lowerCamelCase_    )      ,							Circumscribe(cpu_left_col_base[i + 1]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(gpu_rect[0]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(model_arr[i + 1]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							)
                      if i < 1:
                           self.play(
                               MoveToTarget(model_cpu_arr[i]    )      ,							MoveToTarget(model_cpu_arr[i + 1]    )      ,							)
                      else:
                           self.play(
                               MoveToTarget(model_cpu_arr[i]      ,							run_time=0.7    )      ,							MoveToTarget(model_cpu_arr[i + 1]      ,							run_time=0.7    )      ,							)
                 else:
                      model_cpu_arr[i].generate_target()
                      model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]    )
                      input.generate_target()
                      input.target.next_to(model_arr[-1].get_right()      ,							RIGHT + 0.0_2      ,							buff=0.2    )

                      self.play(
                          Circumscribe(model_arr[-1]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(cpu_left_col_base[-1]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							Circumscribe(gpu_rect[0]      ,							color=lowerCamelCase_      ,							**lowerCamelCase_    )      ,							)

                      self.play(MoveToTarget(model_cpu_arr[i]    )    )

                 a_    :     Optional[Any]										=    a_c
                 a_    :     int										=    a_c.copy()

            input.generate_target()
            input.target.next_to(model_base[-1]      ,							RIGHT + 0.0_2      ,							buff=0.5    )
            self.play(
                FadeOut(lowerCamelCase_    )      ,							FadeOut(lowerCamelCase_      ,							run_time=0.5    )      ,							)

            a_    :     List[str]										=    MarkupText(F'''Inference on a model too large for GPU memory\nis successfully completed.'''      ,							font_size=2_4    )
            step_a.move_to([2, 2, 0]    )

            self.play(Write(lowerCamelCase_      ,							run_time=3    )      ,							MoveToTarget(lowerCamelCase_    )    )

            self.wait()







import os
import re
import shutil
import sys
import tempfile
import unittest

import black


__lowerCamelCase        =     os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
__lowerCamelCase        =     '''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''


class   a__	(						unittest.TestCase    ):

       def 						UpperCAmelCase(   self      :     Dict    ):
            a_    :     Optional[Any]										=    tempfile.mkdtemp()
            os.makedirs(os.path.join(self.transformer_dir      ,							"""models/bert/"""    )    )
            a_    :     Optional[Any]										=    self.transformer_dir
            shutil.copy(
                os.path.join(lowerCamelCase_      ,							"""src/transformers/models/bert/modeling_bert.py"""    )      ,							os.path.join(self.transformer_dir      ,							"""models/bert/modeling_bert.py"""    )      ,							)

       def 						UpperCAmelCase(   self      :     Dict    ):
            a_    :     Any										=    """src/transformers"""
            shutil.rmtree(self.transformer_dir    )

       def 						UpperCAmelCase(   self      :     Dict      ,							lowerCamelCase_      :     str      ,							lowerCamelCase_      :     Optional[int]      ,							lowerCamelCase_      :     int      ,							lowerCamelCase_      :     Tuple=None    ):
            a_    :     Any										=    comment + F'''\nclass {class_name}(nn.Module):\n''' + class_code
            if overwrite_result is not None:
                 a_    :     Any										=    comment + F'''\nclass {class_name}(nn.Module):\n''' + overwrite_result
            a_    :     Dict										=    black.Mode(target_versions={black.TargetVersion.PYaa}      ,							line_length=1_1_9    )
            a_    :     Optional[Any]										=    black.format_str(lowerCamelCase_      ,							mode=lowerCamelCase_    )
            a_    :     Optional[Any]										=    os.path.join(self.transformer_dir      ,							"""new_code.py"""    )
            with open(lowerCamelCase_      ,							"""w"""      ,							newline="""\n"""    ) as f:
                 f.write(lowerCamelCase_    )
            if overwrite_result is None:
                 self.assertTrue(len(check_copies.is_copy_consistent(lowerCamelCase_    )    ) == 0    )
            else:
                 check_copies.is_copy_consistent(f.name      ,							overwrite=lowerCamelCase_    )
                 with open(lowerCamelCase_      ,							"""r"""    ) as f:
                      self.assertTrue(f.read()      ,							lowerCamelCase_    )

       def 						UpperCAmelCase(   self      :     Dict    ):
            a_    :     Tuple										=    check_copies.find_code_in_transformers("""models.bert.modeling_bert.BertLMPredictionHead"""    )
            self.assertEqual(lowerCamelCase_      ,							lowerCamelCase_    )

       def 						UpperCAmelCase(   self      :     Optional[int]    ):
            # Base copy consistency
            self.check_copy_consistency(
                """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""      ,							"""BertLMPredictionHead"""      ,							REFERENCE_CODE + """\n"""      ,							)

            # With no empty line at the end
            self.check_copy_consistency(
                """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""      ,							"""BertLMPredictionHead"""      ,							lowerCamelCase_      ,							)

            # Copy consistency with rename
            self.check_copy_consistency(
                """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""      ,							"""TestModelLMPredictionHead"""      ,							re.sub("""Bert"""      ,							"""TestModel"""      ,							lowerCamelCase_    )      ,							)

            # Copy consistency with a really long name
            a_    :     int										=    """TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
            self.check_copy_consistency(
                F'''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'''      ,							F'''{long_class_name}LMPredictionHead'''      ,							re.sub("""Bert"""      ,							lowerCamelCase_      ,							lowerCamelCase_    )      ,							)

            # Copy consistency with overwrite
            self.check_copy_consistency(
                """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""      ,							"""TestModelLMPredictionHead"""      ,							lowerCamelCase_      ,							overwrite_result=re.sub("""Bert"""      ,							"""TestModel"""      ,							lowerCamelCase_    )      ,							)

       def 						UpperCAmelCase(   self      :     int    ):
            a_    :     Optional[Any]										=    check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

            a_    :     Tuple										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
                """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
                """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
                """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
                """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
                """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
                """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
                """ method has been applied to compress GPT2 into"""
                """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
                """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
                """ Multilingual BERT into"""
                """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
                """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
                """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
                """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
                """ Luong, Quoc V. Le, Christopher D. Manning."""
            )
            a_    :     Optional[Any]										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
            )
            a_    :     Any										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
                """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
                """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
                """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
                """ method has been applied to compress GPT2 into"""
                """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
                """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
                """ Multilingual BERT into"""
                """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
                """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
                """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
                """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
                """ Christopher D. Manning 发布。\n"""
            )

            a_			,			a_    :     Optional[int]										=    check_copies.convert_to_localized_md(
                lowerCamelCase_      ,							lowerCamelCase_      ,							localized_readme["""format_model_list"""]    )

            self.assertFalse(lowerCamelCase_    )
            self.assertEqual(lowerCamelCase_      ,							lowerCamelCase_    )

            a_			,			a_    :     Tuple										=    check_copies.convert_to_localized_md(
                lowerCamelCase_      ,							lowerCamelCase_      ,							localized_readme["""format_model_list"""]    )

            # Check whether the number of models is equal to README.md after conversion.
            self.assertTrue(lowerCamelCase_    )

            a_    :     Optional[int]										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
                """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
                """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
                """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
            )
            a_    :     Union[str, Any]										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
                """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
            )
            a_    :     str										=    (
                """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
                """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
                """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
                """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
            )

            a_			,			a_    :     Tuple										=    check_copies.convert_to_localized_md(
                lowerCamelCase_      ,							lowerCamelCase_      ,							localized_readme["""format_model_list"""]    )

            # Check if the model link is synchronized.
            self.assertEqual(lowerCamelCase_      ,							lowerCamelCase_    )



