



import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
   if model.config.model_type == "gpt2":
      return model.transformer.h[0].mlp.c_fc
   return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
       import torch
       import torch.nn as nn
       class 				_snake_case			(					nn.Module):




           def __init__(						self					:						Union[str, Any],					__lowercase					:						nn.Module,					__lowercase					:						int						):
              super().__init__()
              lowercase__					  =				module
              lowercase__					  =				nn.Sequential(
                  nn.Linear(module.in_features,					__lowercase,					bias=__lowercase						),					nn.Linear(__lowercase,					module.out_features,					bias=__lowercase						),					)
              lowercase__					  =				(2.0 / (5 * min(module.in_features,					module.out_features						))) ** 0.5
              nn.init.normal_(self.adapter[0].weight,					std=__lowercase						)
              nn.init.zeros_(self.adapter[1].weight						)
              self.adapter.to(module.weight.device						)




           def 							A__      (						self					:						List[str],					__lowercase					:						Any,					*__lowercase					:						Any,					**__lowercase					:						Dict						):
              return self.module(__lowercase,					*__lowercase,					**__lowercase						) + self.adapter(__lowercase						)
@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				_snake_case			(					unittest.TestCase):
    # We keep the constants inside the init function and model loading inside setUp function

    # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
    # Therefore here we use only bloom-1b3 to test our module
    UpperCamelCase__				:					int     		="""bigscience/bloom-1b7"""

    # Constant values
    UpperCamelCase__				:					List[str]     		=2.109_659_552_692_574

    UpperCamelCase__				:					str     		="""Hello my name is"""
    UpperCamelCase__				:					List[Any]     		=set()
    EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I""")
    EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n""")
    EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University""")
    UpperCamelCase__				:					Optional[Any]     		=1_0




    def 							A__      (						self					:						Optional[int]						):
       # Models and tokenizer
       lowercase__					  =				AutoTokenizer.from_pretrained(self.model_name						)
class 				_snake_case			(					lowercase__):




    def 							A__      (						self					:						Union[str, Any]						):
       super().setUp()

       # Models and tokenizer
       lowercase__					  =				AutoModelForCausalLM.from_pretrained(
           self.model_name,					torch_dtype=torch.floataa,					device_map="auto"						)
       lowercase__					  =				AutoModelForCausalLM.from_pretrained(self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)




    def 							A__      (						self					:						Union[str, Any]						):
       del self.model_fpaa
       del self.model_abit

       gc.collect()
       torch.cuda.empty_cache()




    def 							A__      (						self					:						int						):
       lowercase__					  =				self.model_abit.config

       self.assertTrue(hasattr(__lowercase,					"quantization_config"						)						)

       lowercase__					  =				config.to_dict()
       lowercase__					  =				config.to_diff_dict()

       lowercase__					  =				config.to_json_string()




    def 							A__      (						self					:						List[str]						):
       from bitsandbytes.nn import Paramsabit

       lowercase__					  =				self.model_fpaa.get_memory_footprint()
       lowercase__					  =				self.model_abit.get_memory_footprint()

       self.assertAlmostEqual(mem_fpaa / mem_abit,					self.EXPECTED_RELATIVE_DIFFERENCE						)
       lowercase__					  =				get_some_linear_layer(self.model_abit						)
       self.assertTrue(linear.weight.__class__ == Paramsabit						)




    def 							A__      (						self					:						Optional[Any]						):
       from transformers import TaPreTrainedModel

       self.model_fpaa.get_memory_footprint()
       self.model_abit.get_memory_footprint()

       for name, module in self.model_abit.named_modules():
          if isinstance(__lowercase,					torch.nn.Linear						):
             if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                # 4-bit parameters are packed in uint8 variables
                self.assertTrue(module.weight.dtype == torch.uinta						)




    def 							A__      (						self					:						Optional[int]						):
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						)
       lowercase__					  =				self.model_abit.generate(input_ids=encoded_input["input_ids"].to(0						),					max_new_tokens=10						)

       self.assertIn(self.tokenizer.decode(output_sequences[0],					skip_special_tokens=__lowercase						),					self.EXPECTED_OUTPUTS						)




    def 							A__      (						self					:						str						):
       lowercase__					  =				BitsAndBytesConfig()
       lowercase__					  =				True

       lowercase__					  =				AutoModelForCausalLM.from_pretrained(
           self.model_name,					quantization_config=__lowercase,					device_map="auto"						)

       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						)
       lowercase__					  =				model_abit_from_config.generate(
           input_ids=encoded_input["input_ids"].to(0						),					max_new_tokens=10						)

       self.assertIn(self.tokenizer.decode(output_sequences[0],					skip_special_tokens=__lowercase						),					self.EXPECTED_OUTPUTS						)




    def 							A__      (						self					:						int						):
       with self.assertRaises(__lowercase						), tempfile.TemporaryDirectory() as tmpdirname:
          self.model_abit.save_pretrained(__lowercase						)




    def 							A__      (						self					:						str						):
       lowercase__					  =				BitsAndBytesConfig()

       with self.assertRaises(__lowercase						):
          lowercase__					  =				AutoModelForCausalLM.from_pretrained(
              self.model_name,					quantization_config=__lowercase,					load_in_abit=__lowercase,					device_map="auto",					bnb_abit_quant_type="nf4",					)




    def 							A__      (						self					:						Tuple						):
       with self.assertRaises(__lowercase						):
          # Tries with `str`
          self.model_abit.to("cpu"						)

       with self.assertRaises(__lowercase						):
          # Tries with a `dtype``
          self.model_abit.to(torch.floataa						)

       with self.assertRaises(__lowercase						):
          # Tries with a `device`
          self.model_abit.to(torch.device("cuda:0"						)						)

       with self.assertRaises(__lowercase						):
          # Tries with a `device`
          self.model_abit.float()

       with self.assertRaises(__lowercase						):
          # Tries with a `device`
          self.model_abit.half()

       # Test if we did not break anything
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						)

       lowercase__					  =				self.model_fpaa.to(torch.floataa						)
       lowercase__					  =				self.model_fpaa.generate(input_ids=encoded_input["input_ids"].to(0						),					max_new_tokens=10						)

       # Check this does not throw an error
       lowercase__					  =				self.model_fpaa.to("cpu"						)

       # Check this does not throw an error
       lowercase__					  =				self.model_fpaa.half()

       # Check this does not throw an error
       lowercase__					  =				self.model_fpaa.float()




    def 							A__      (						self					:						Any						):
       lowercase__					  =				AutoModelForSeqaSeqLM.from_pretrained("t5-small",					load_in_abit=__lowercase,					device_map="auto"						)
       self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa						)
@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				_snake_case			(					unittest.TestCase):




    @classmethod
    def 							A__      (						cls					:						Union[str, Any]						):
       lowercase__					  =				"t5-small"
       lowercase__					  =				"google/flan-t5-small"  # flan-t5 uses dense-act instead of dense-relu-dense
       lowercase__					  =				AutoTokenizer.from_pretrained(cls.model_name						)
       lowercase__					  =				"Translate in German: Hello, my dog is cute"




    def 							A__      (						self					:						List[str]						):
       gc.collect()
       torch.cuda.empty_cache()




    def 							A__      (						self					:						Dict						):
       from transformers import TaForConditionalGeneration

       lowercase__					  =				TaForConditionalGeneration._keep_in_fpaa_modules
       lowercase__					  =				None

       # test with `t5-small`
       lowercase__					  =				TaForConditionalGeneration.from_pretrained(self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						).to(0						)
       lowercase__					  =				model.generate(**__lowercase						)

       # test with `flan-t5-small`
       lowercase__					  =				TaForConditionalGeneration.from_pretrained(
           self.dense_act_model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						).to(0						)
       lowercase__					  =				model.generate(**__lowercase						)
       lowercase__					  =				modules




    def 							A__      (						self					:						int						):
       import bitsandbytes as bnb

       from transformers import TaForConditionalGeneration

       # test with `t5-small`
       lowercase__					  =				TaForConditionalGeneration.from_pretrained(self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)

       # there was a bug with decoders - this test checks that it is fixed
       self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q,					bnb.nn.Linearabit						)						)

       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						).to(0						)
       lowercase__					  =				model.generate(**__lowercase						)

       # test with `flan-t5-small`
       lowercase__					  =				TaForConditionalGeneration.from_pretrained(
           self.dense_act_model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						).to(0						)
       lowercase__					  =				model.generate(**__lowercase						)
class 				_snake_case			(					lowercase__):




    def 							A__      (						self					:						Tuple						):
       super().setUp()
       # model_name
       lowercase__					  =				"bigscience/bloom-560m"
       lowercase__					  =				"t5-small"

       # Different types of model

       lowercase__					  =				AutoModel.from_pretrained(self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       # Sequence classification model
       lowercase__					  =				AutoModelForSequenceClassification.from_pretrained(
           self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       # CausalLM model
       lowercase__					  =				AutoModelForCausalLM.from_pretrained(self.model_name,					load_in_abit=__lowercase,					device_map="auto"						)
       # Seq2seq model
       lowercase__					  =				AutoModelForSeqaSeqLM.from_pretrained(
           self.seq_to_seq_name,					load_in_abit=__lowercase,					device_map="auto"						)




    def 							A__      (						self					:						List[Any]						):
       del self.base_model
       del self.sequence_model
       del self.model_abit
       del self.seq_to_seq_model

       gc.collect()
       torch.cuda.empty_cache()




    def 							A__      (						self					:						Optional[int]						):
       from bitsandbytes.nn import Paramsabit

       self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit						)

       # Other heads should be nn.Parameter
       self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter						)
       self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter						)
       self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter						)
class 				_snake_case			(					lowercase__):




    def 							A__      (						self					:						Tuple						):
       super().setUp()




    def 							A__      (						self					:						List[Any]						):
       del self.pipe

       gc.collect()
       torch.cuda.empty_cache()




    def 							A__      (						self					:						Union[str, Any]						):
       lowercase__					  =				pipeline(
           "text-generation",					model=self.model_name,					model_kwargs={"device_map": "auto", "load_in_4bit": True, "torch_dtype": torch.floataa},					max_new_tokens=self.MAX_NEW_TOKENS,					)

       # Real second forward pass
       lowercase__					  =				self.pipe(self.input_text						)
       self.assertIn(pipeline_output[0]["generated_text"],					self.EXPECTED_OUTPUTS						)
@require_torch_multi_gpu
class 				_snake_case			(					lowercase__):




    def 							A__      (						self					:						Any						):
       super().setUp()




    def 							A__      (						self					:						str						):
       lowercase__					  =				AutoModelForCausalLM.from_pretrained(
           self.model_name,					load_in_abit=__lowercase,					device_map="balanced"						)

       # Check correct device map
       self.assertEqual(set(model_parallel.hf_device_map.values()						),					{0, 1}						)

       # Check that inference pass works on the model
       lowercase__					  =				self.tokenizer(self.input_text,					return_tensors="pt"						)

       # Second real batch
       lowercase__					  =				model_parallel.generate(input_ids=encoded_input["input_ids"].to(0						),					max_new_tokens=10						)
       self.assertIn(self.tokenizer.decode(output_parallel[0],					skip_special_tokens=__lowercase						),					self.EXPECTED_OUTPUTS						)
class 				_snake_case			(					lowercase__):




    def 							A__      (						self					:						Optional[Any]						):
       lowercase__					  =				"facebook/opt-350m"
       super().setUp()




    def 							A__      (						self					:						Dict						):
       if version.parse(importlib.metadata.version("bitsandbytes"						)						) < version.parse("0.37.0"						):
          return

       # Step 1: freeze all parameters
       lowercase__					  =				AutoModelForCausalLM.from_pretrained(self.model_name,					load_in_abit=__lowercase						)

       self.assertEqual(set(model.hf_device_map.values()						),					{torch.cuda.current_device()}						)

       for param in model.parameters():
          lowercase__					  =				False  # freeze the model - train adapters later
          if param.ndim == 1:
             # cast the small parameters (e.g. layernorm) to fp32 for stability
             lowercase__					  =				param.data.to(torch.floataa						)

        # Step 2: add adapters
       for _, module in model.named_modules():
          if "OPTAttention" in repr(type(__lowercase						)						):
             lowercase__					  =				LoRALayer(module.q_proj,					rank=16						)
             lowercase__					  =				LoRALayer(module.k_proj,					rank=16						)
             lowercase__					  =				LoRALayer(module.v_proj,					rank=16						)

        # Step 3: dummy batch
       lowercase__					  =				self.tokenizer("Test batch ",					return_tensors="pt"						).to(0						)

       # Step 4: Check if the gradient is not None
       with torch.cuda.amp.autocast():
          lowercase__					  =				model.forward(**__lowercase						)
          out.logits.norm().backward()

       for module in model.modules():
          if isinstance(__lowercase,					__lowercase						):
             self.assertTrue(module.adapter[1].weight.grad is not None						)
             self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0						)
          elif isinstance(__lowercase,					nn.Embedding						):
             self.assertTrue(module.weight.grad is None						)
class 				_snake_case			(					lowercase__):
    UpperCamelCase__				:					Union[str, Any]     		="""gpt2-xl"""
    UpperCamelCase__				:					Any     		=3.3_191_854_854_152_187









import pickle

import numpy as np
from matplotlib import pyplot as plt
class 				_snake_case			:




				def __init__(						self					:						Tuple,					__lowercase					:						Union[str, Any],					__lowercase					:						int,					__lowercase					:						Union[str, Any],					__lowercase					:						str,					__lowercase					:						List[Any],					__lowercase					:						List[str]=0.2,					__lowercase					:						List[str]=0.2						):
							lowercase__					  =				bp_numa
							lowercase__					  =				bp_numa
							lowercase__					  =				bp_numa
							lowercase__					  =				conva_get[:2]
							lowercase__					  =				conva_get[2]
							lowercase__					  =				size_pa
							lowercase__					  =				rate_w
							lowercase__					  =				rate_t
							lowercase__					  =				[
							    np.mat(-1 * np.random.rand(self.conva[0],					self.conva[0]						) + 0.5						)
							    for i in range(self.conva[1]						)
							]
							lowercase__					  =				np.mat(-1 * np.random.rand(self.num_bpa,					self.num_bpa						) + 0.5						)
							lowercase__					  =				np.mat(-1 * np.random.rand(self.num_bpa,					self.num_bpa						) + 0.5						)
							lowercase__					  =				-2 * np.random.rand(self.conva[1]						) + 1
							lowercase__					  =				-2 * np.random.rand(self.num_bpa						) + 1
							lowercase__					  =				-2 * np.random.rand(self.num_bpa						) + 1




				def 							A__      (						self					:						Any,					__lowercase					:						List[str]						):
							# save model dict with pickle
							lowercase__					  =				{
							    "num_bp1": self.num_bpa,
							    "num_bp2": self.num_bpa,
							    "num_bp3": self.num_bpa,
							    "conv1": self.conva,
							    "step_conv1": self.step_conva,
							    "size_pooling1": self.size_poolinga,
							    "rate_weight": self.rate_weight,
							    "rate_thre": self.rate_thre,
							    "w_conv1": self.w_conva,
							    "wkj": self.wkj,
							    "vji": self.vji,
							    "thre_conv1": self.thre_conva,
							    "thre_bp2": self.thre_bpa,
							    "thre_bp3": self.thre_bpa,
							}
							with open(__lowercase,					"wb"						) as f:
										pickle.dump(__lowercase,					__lowercase						)

							print(F'''Model saved： {save_path}'''						)




				@classmethod
				def 							A__      (						cls					:						Dict,					__lowercase					:						Union[str, Any]						):
							# read saved model
							with open(__lowercase,					"rb"						) as f:
										lowercase__					  =				pickle.load(__lowercase						)  # noqa: S301

							lowercase__					  =				model_dic.get("conv1"						)
							conv_get.append(model_dic.get("step_conv1"						)						)
							lowercase__					  =				model_dic.get("size_pooling1"						)
							lowercase__					  =				model_dic.get("num_bp1"						)
							lowercase__					  =				model_dic.get("num_bp2"						)
							lowercase__					  =				model_dic.get("num_bp3"						)
							lowercase__					  =				model_dic.get("rate_weight"						)
							lowercase__					  =				model_dic.get("rate_thre"						)
							# create model instance
							lowercase__					  =				CNN(__lowercase,					__lowercase,					__lowercase,					__lowercase,					__lowercase,					__lowercase,					__lowercase						)
							# modify model parameter
							lowercase__					  =				model_dic.get("w_conv1"						)
							lowercase__					  =				model_dic.get("wkj"						)
							lowercase__					  =				model_dic.get("vji"						)
							lowercase__					  =				model_dic.get("thre_conv1"						)
							lowercase__					  =				model_dic.get("thre_bp2"						)
							lowercase__					  =				model_dic.get("thre_bp3"						)
							return conv_ins




				def 							A__      (						self					:						str,					__lowercase					:						List[Any]						):
							return 1 / (1 + np.exp(-1 * x						))




				def 							A__      (						self					:						List[str],					__lowercase					:						Optional[Any]						):
							return round(__lowercase,					3						)




				def 							A__      (						self					:						Optional[Any],					__lowercase					:						Dict,					__lowercase					:						Optional[int],					__lowercase					:						Optional[int],					__lowercase					:						Optional[Any],					__lowercase					:						str						):
							# convolution process
							lowercase__					  =				convs[0]
							lowercase__					  =				convs[1]
							lowercase__					  =				np.shape(__lowercase						)[0]
							# get the data slice of original image data, data_focus
							lowercase__					  =				[]
							for i_focus in range(0,					size_data - size_conv + 1,					__lowercase						):
										for j_focus in range(0,					size_data - size_conv + 1,					__lowercase						):
													lowercase__					  =				data[
													    i_focus : i_focus + size_conv, j_focus : j_focus + size_conv
													]
													data_focus.append(__lowercase						)
        # calculate the feature map of every single kernel, and saved as list of matrix
							lowercase__					  =				[]
							lowercase__					  =				int((size_data - size_conv) / conv_step + 1						)
							for i_map in range(__lowercase						):
										lowercase__					  =				[]
										for i_focus in range(len(__lowercase						)						):
													lowercase__					  =				(
													    np.sum(np.multiply(data_focus[i_focus],					w_convs[i_map]						)						)
													    - thre_convs[i_map]
													)
													featuremap.append(self.sig(__lowercase						)						)
										lowercase__					  =				np.asmatrix(__lowercase						).reshape(
										    __lowercase,					__lowercase						)
										data_featuremap.append(__lowercase						)

							# expanding the data slice to One dimenssion
							lowercase__					  =				[]
							for each_focus in data_focus:
										focusa_list.extend(self.Expand_Mat(__lowercase						)						)
							lowercase__					  =				np.asarray(__lowercase						)
							return focus_list, data_featuremap




				def 							A__      (						self					:						List[Any],					__lowercase					:						Any,					__lowercase					:						List[Any],					__lowercase					:						Union[str, Any]="average_pool"						):
							# pooling process
							lowercase__					  =				len(featuremaps[0]						)
							lowercase__					  =				int(size_map / size_pooling						)
							lowercase__					  =				[]
							for i_map in range(len(__lowercase						)						):
										lowercase__					  =				featuremaps[i_map]
										lowercase__					  =				[]
										for i_focus in range(0,					__lowercase,					__lowercase						):
													for j_focus in range(0,					__lowercase,					__lowercase						):
																lowercase__					  =				feature_map[
																    i_focus : i_focus + size_pooling,
																    j_focus : j_focus + size_pooling,
																]
																if pooling_type == "average_pool":
																			# average pooling
																			map_pooled.append(np.average(__lowercase						)						)
																elif pooling_type == "max_pooling":
																			# max pooling
																			map_pooled.append(np.max(__lowercase						)						)
										lowercase__					  =				np.asmatrix(__lowercase						).reshape(__lowercase,					__lowercase						)
										featuremap_pooled.append(__lowercase						)
							return featuremap_pooled




				def 							A__      (						self					:						str,					__lowercase					:						Optional[Any]						):
							# expanding three dimension data to one dimension list
							lowercase__					  =				[]
							for i in range(len(__lowercase						)						):
										lowercase__					  =				np.shape(data[i]						)
										lowercase__					  =				data[i].reshape(1,					shapes[0] * shapes[1]						)
										lowercase__					  =				data_listed.getA().tolist()[0]
										data_expanded.extend(__lowercase						)
							lowercase__					  =				np.asarray(__lowercase						)
							return data_expanded




				def 							A__      (						self					:						Optional[int],					__lowercase					:						Optional[int]						):
							# expanding matrix to one dimension list
							lowercase__					  =				np.asarray(__lowercase						)
							lowercase__					  =				np.shape(__lowercase						)
							lowercase__					  =				data_mat.reshape(1,					shapes[0] * shapes[1]						)
							return data_expanded




				def 							A__      (						self					:						str,					__lowercase					:						Tuple,					__lowercase					:						List[Any],					__lowercase					:						Any,					__lowercase					:						Union[str, Any],					__lowercase					:						Tuple						):
							lowercase__					  =				[]
							lowercase__					  =				0
							for i_map in range(__lowercase						):
										lowercase__					  =				np.ones((size_map, size_map)						)
										for i in range(0,					__lowercase,					__lowercase						):
													for j in range(0,					__lowercase,					__lowercase						):
																lowercase__					  =				pd_pool[
																    i_pool
																]
																lowercase__					  =				i_pool + 1
										lowercase__					  =				np.multiply(
										    __lowercase,					np.multiply(out_map[i_map],					(1 - out_map[i_map])						)						)
										pd_all.append(__lowercase						)
							return pd_all




				def 							A__      (						self					:						Tuple,					__lowercase					:						int,					__lowercase					:						Optional[Any],					__lowercase					:						List[Any],					__lowercase					:						Optional[Any],					__lowercase					:						List[Any],					__lowercase					:						List[str]=bool						):
							# model traning
							print("----------------------Start Training-------------------------"						)
							print((" - - Shape: Train_Data  ", np.shape(__lowercase						))						)
							print((" - - Shape: Teach_Data  ", np.shape(__lowercase						))						)
							lowercase__					  =				0
							lowercase__					  =				[]
							lowercase__					  =				1_0000
							while rp < n_repeat and mse >= error_accuracy:
										lowercase__					  =				0
										print(F'''-------------Learning Time {rp}--------------'''						)
										for p in range(len(__lowercase						)						):
													# print('------------Learning Image: %d--------------'%p)
													lowercase__					  =				np.asmatrix(datas_train[p]						)
													lowercase__					  =				np.asarray(datas_teach[p]						)
													lowercase__  ,       lowercase__					  =				self.convolute(
													    __lowercase,					self.conva,					self.w_conva,					self.thre_conva,					conv_step=self.step_conva,					)
													lowercase__					  =				self.pooling(__lowercase,					self.size_poolinga						)
													lowercase__					  =				np.shape(__lowercase						)
													lowercase__					  =				self._expand(__lowercase						)
													lowercase__					  =				data_bp_input

													lowercase__					  =				np.dot(__lowercase,					self.vji.T						) - self.thre_bpa
													lowercase__					  =				self.sig(__lowercase						)
													lowercase__					  =				np.dot(__lowercase,					self.wkj.T						) - self.thre_bpa
													lowercase__					  =				self.sig(__lowercase						)

													# --------------Model Leaning ------------------------
													# calculate error and gradient---------------
													lowercase__					  =				np.multiply(
													    (data_teach - bp_outa),					np.multiply(__lowercase,					(1 - bp_outa)						)						)
													lowercase__					  =				np.multiply(
													    np.dot(__lowercase,					self.wkj						),					np.multiply(__lowercase,					(1 - bp_outa)						)						)
													lowercase__					  =				np.dot(__lowercase,					self.vji						)

													lowercase__					  =				pd_i_all / (self.size_poolinga * self.size_poolinga)
													lowercase__					  =				pd_conva_pooled.T.getA().tolist()
													lowercase__					  =				self._calculate_gradient_from_pool(
													    __lowercase,					__lowercase,					shape_featuremapa[0],					shape_featuremapa[1],					self.size_poolinga,					)
													# weight and threshold learning process---------
													# convolution layer
													for k_conv in range(self.conva[1]						):
																lowercase__					  =				self._expand_mat(pd_conva_all[k_conv]						)
																lowercase__					  =				self.rate_weight * np.dot(__lowercase,					__lowercase						)

																lowercase__					  =				self.w_conva[k_conv] + delta_w.reshape(
																    (self.conva[0], self.conva[0])						)

																lowercase__					  =				(
																    self.thre_conva[k_conv]
																    - np.sum(pd_conva_all[k_conv]						) * self.rate_thre
																)
													# all connected layer
													lowercase__					  =				self.wkj + pd_k_all.T * bp_outa * self.rate_weight
													lowercase__					  =				self.vji + pd_j_all.T * bp_outa * self.rate_weight
													lowercase__					  =				self.thre_bpa - pd_k_all * self.rate_thre
													lowercase__					  =				self.thre_bpa - pd_j_all * self.rate_thre
													# calculate the sum error of all single image
													lowercase__					  =				np.sum(abs(data_teach - bp_outa						)						)
													error_count += errors
													# print('   ----Teach      ',data_teach)
													# print('   ----BP_output  ',bp_out3)
										lowercase__					  =				rp + 1
										lowercase__					  =				error_count / patterns
										all_mse.append(__lowercase						)

							def draw_error():
										lowercase__					  =				[error_accuracy for i in range(int(n_repeat * 1.2						)						)]
										plt.plot(__lowercase,					"+-"						)
										plt.plot(__lowercase,					"r--"						)
										plt.xlabel("Learning Times"						)
										plt.ylabel("All_mse"						)
										plt.grid(__lowercase,					alpha=0.5						)
										plt.show()

							print("------------------Training Complished---------------------"						)
							print((" - - Training epoch: ", rp, F'''     - - Mse: {mse:.6f}''')						)
							if draw_e:
										draw_error()
							return mse




				def 							A__      (						self					:						List[str],					__lowercase					:						Optional[int]						):
							# model predict
							lowercase__					  =				[]
							print("-------------------Start Testing-------------------------"						)
							print((" - - Shape: Test_Data  ", np.shape(__lowercase						))						)
							for p in range(len(__lowercase						)						):
										lowercase__					  =				np.asmatrix(datas_test[p]						)
										lowercase__  ,       lowercase__					  =				self.convolute(
										    __lowercase,					self.conva,					self.w_conva,					self.thre_conva,					conv_step=self.step_conva,					)
										lowercase__					  =				self.pooling(__lowercase,					self.size_poolinga						)
										lowercase__					  =				self._expand(__lowercase						)

										lowercase__					  =				data_bp_input
										lowercase__					  =				bp_outa * self.vji.T - self.thre_bpa
										lowercase__					  =				self.sig(__lowercase						)
										lowercase__					  =				bp_outa * self.wkj.T - self.thre_bpa
										lowercase__					  =				self.sig(__lowercase						)
										produce_out.extend(bp_outa.getA().tolist()						)
							lowercase__					  =				[list(map(self.do_round,					__lowercase						)						) for each in produce_out]
							return np.asarray(__lowercase						)




				def 							A__      (						self					:						int,					__lowercase					:						Any						):
							# return the data of image after convoluting process so we can check it out
							lowercase__					  =				np.asmatrix(__lowercase						)
							lowercase__  ,       lowercase__					  =				self.convolute(
							    __lowercase,					self.conva,					self.w_conva,					self.thre_conva,					conv_step=self.step_conva,					)
							lowercase__					  =				self.pooling(__lowercase,					self.size_poolinga						)

							return data_conveda, data_pooleda


if __name__ == "__main__":
							pass




