
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase      =logging.get_logger(__name__)

lowerCamelCase      ={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase      ={
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowerCamelCase      ={
    "bert-base-uncased": 5_1_2,
    "bert-large-uncased": 5_1_2,
    "bert-base-cased": 5_1_2,
    "bert-large-cased": 5_1_2,
    "bert-base-multilingual-uncased": 5_1_2,
    "bert-base-multilingual-cased": 5_1_2,
    "bert-base-chinese": 5_1_2,
    "bert-base-german-cased": 5_1_2,
    "bert-large-uncased-whole-word-masking": 5_1_2,
    "bert-large-cased-whole-word-masking": 5_1_2,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-base-cased-finetuned-mrpc": 5_1_2,
    "bert-base-german-dbmdz-cased": 5_1_2,
    "bert-base-german-dbmdz-uncased": 5_1_2,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_1_2,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_1_2,
    "wietsedv/bert-base-dutch-cased": 5_1_2,
}

lowerCamelCase      ={
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}




class 				_lowerCamelCase (      UpperCamelCase_ ):





					"""simple docstring"""







					SCREAMING_SNAKE_CASE_		   =  VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE_		   =  PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE_		   =  PRETRAINED_INIT_CONFIGURATION
					SCREAMING_SNAKE_CASE_		   =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE_		   =  BertTokenizer






					def __init__(							self						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=None						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE="[UNK]"						,						__SCREAMING_SNAKE_CASE="[SEP]"						,						__SCREAMING_SNAKE_CASE="[PAD]"						,						__SCREAMING_SNAKE_CASE="[CLS]"						,						__SCREAMING_SNAKE_CASE="[MASK]"						,						__SCREAMING_SNAKE_CASE=True						,						__SCREAMING_SNAKE_CASE=None						,						**__SCREAMING_SNAKE_CASE						,						)  ->   List[Any]:




											"""simple docstring"""







											super().__init__(
											    __SCREAMING_SNAKE_CASE						,						tokenizer_file=__SCREAMING_SNAKE_CASE						,						do_lower_case=__SCREAMING_SNAKE_CASE						,						unk_token=__SCREAMING_SNAKE_CASE						,						sep_token=__SCREAMING_SNAKE_CASE						,						pad_token=__SCREAMING_SNAKE_CASE						,						cls_token=__SCREAMING_SNAKE_CASE						,						mask_token=__SCREAMING_SNAKE_CASE						,						tokenize_chinese_chars=__SCREAMING_SNAKE_CASE						,						strip_accents=__SCREAMING_SNAKE_CASE						,						**__SCREAMING_SNAKE_CASE						,						)

											UpperCamelCase__    :   Dict					=				json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
											if (
											    normalizer_state.get('''lowercase'''						,						__SCREAMING_SNAKE_CASE  ) != do_lower_case
											    or normalizer_state.get('''strip_accents'''						,						__SCREAMING_SNAKE_CASE  ) != strip_accents
											    or normalizer_state.get('''handle_chinese_chars'''						,						__SCREAMING_SNAKE_CASE  ) != tokenize_chinese_chars
											):
																	UpperCamelCase__    :   Union[str, Any]					=				getattr(__SCREAMING_SNAKE_CASE						,						normalizer_state.pop('''type'''  )  )
																	UpperCamelCase__    :   int					=				do_lower_case
																	UpperCamelCase__    :   Optional[Any]					=				strip_accents
																	UpperCamelCase__    :   List[Any]					=				tokenize_chinese_chars
																	UpperCamelCase__    :   int					=				normalizer_class(**__SCREAMING_SNAKE_CASE  )

											UpperCamelCase__    :   Optional[int]					=				do_lower_case






					def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE=None  )  ->   List[str]:




											"""simple docstring"""







											UpperCamelCase__    :   Dict					=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

											return output






					def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE = None  )  ->   List[int]:




											"""simple docstring"""







											UpperCamelCase__    :   str					=				[self.sep_token_id]
											UpperCamelCase__    :   int					=				[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep  ) * [0]
											return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






					def    __SCREAMING_SNAKE_CASE			(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE = None  )  ->   Tuple[str]:




											"""simple docstring"""







											UpperCamelCase__    :   Tuple					=				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE						,						name=__SCREAMING_SNAKE_CASE  )
											return tuple(__SCREAMING_SNAKE_CASE  )




from argparse import ArgumentParser

from ..pipelines import Pipeline, PipelineDataFormat, get_supported_tasks, pipeline
from ..utils import logging
from . import BaseTransformersCLICommand


lowerCamelCase      =logging.get_logger(__name__)  # pylint: disable=invalid-name


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__		):
      if not path:
            return "pipe"

      for ext in PipelineDataFormat.SUPPORTED_FORMATS:
            if path.endswith(UpperCamelCase__		):
                  return ext

      raise Exception(
          f'''Unable to determine file format from file extension {path}. '''
          f'''Please provide the format through --format {PipelineDataFormat.SUPPORTED_FORMATS}'''		)


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__		):
      UpperCamelCase__    :   Tuple					=				pipeline(
          task=args.task ,		model=args.model if args.model else None ,		config=args.config ,		tokenizer=args.tokenizer ,		device=args.device ,		)
      UpperCamelCase__    :   Any					=				try_infer_format_from_ext(args.input		) if args.format == '''infer''' else args.format
      UpperCamelCase__    :   Tuple					=				PipelineDataFormat.from_str(
          format=UpperCamelCase__ ,		output_path=args.output ,		input_path=args.input ,		column=args.column if args.column else nlp.default_input_names ,		overwrite=args.overwrite ,		)
      return RunCommand(UpperCamelCase__ ,		UpperCamelCase__		)




class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     def __init__(							self						,						__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				nlp
           UpperCamelCase__    :   Dict					=				reader






     @staticmethod
     def    __SCREAMING_SNAKE_CASE			(							__SCREAMING_SNAKE_CASE  )  ->   str:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				parser.add_parser('''run'''						,						help='''Run a pipeline through the CLI'''  )
           run_parser.add_argument('''--task'''						,						choices=get_supported_tasks()						,						help='''Task to run'''  )
           run_parser.add_argument('''--input'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Path to the file to use for inference'''  )
           run_parser.add_argument('''--output'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Path to the file that will be used post to write results.'''  )
           run_parser.add_argument('''--model'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Name or path to the model to instantiate.'''  )
           run_parser.add_argument('''--config'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Name or path to the model\'s config to instantiate.'''  )
           run_parser.add_argument(
               '''--tokenizer'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Name of the tokenizer to use. (default: same as the model name)'''  )
           run_parser.add_argument(
               '''--column'''						,						type=__SCREAMING_SNAKE_CASE						,						help='''Name of the column to use as input. (For multi columns input as QA use column1,columns2)'''						,						)
           run_parser.add_argument(
               '''--format'''						,						type=__SCREAMING_SNAKE_CASE						,						default='''infer'''						,						choices=PipelineDataFormat.SUPPORTED_FORMATS						,						help='''Input format to read from'''						,						)
           run_parser.add_argument(
               '''--device'''						,						type=__SCREAMING_SNAKE_CASE						,						default=-1						,						help='''Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)'''						,						)
           run_parser.add_argument('''--overwrite'''						,						action='''store_true'''						,						help='''Allow overwriting the output file.'''  )
           run_parser.set_defaults(func=__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           UpperCamelCase__				,UpperCamelCase__    :   Optional[Any]					=				self._nlp, []

           for entry in self._reader:
                 UpperCamelCase__    :   List[str]					=				nlp(**__SCREAMING_SNAKE_CASE  ) if self._reader.is_multi_columns else nlp(__SCREAMING_SNAKE_CASE  )
                 if isinstance(__SCREAMING_SNAKE_CASE						,						__SCREAMING_SNAKE_CASE  ):
                       outputs.append(__SCREAMING_SNAKE_CASE  )
                 else:
                       outputs += output

        # Saving data
           if self._nlp.binary_output:
                 UpperCamelCase__    :   Union[str, Any]					=				self._reader.save_binary(__SCREAMING_SNAKE_CASE  )
                 logger.warning(F'''Current pipeline requires output to be in binary format, saving at {binary_path}'''  )
           else:
                 self._reader.save(__SCREAMING_SNAKE_CASE  )


