





"""simple docstring"""






import json
import logging
import math
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from datasets import Dataset, load_dataset

import transformers
from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_MASKED_LM_MAPPING,
    AutoConfig,
    AutoModelForMaskedLM,
    AutoTokenizer,
    DataCollatorForWholeWordMask,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


lowerCAmelCase_			=							logging.getLogger(__name__)
lowerCAmelCase_			=							list(MODEL_FOR_MASKED_LM_MAPPING.keys())
lowerCAmelCase_			=							tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)



@dataclass
class 	lowerCAmelCase					:
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={
               """help""": (
                   """The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."""
               )
           }			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """If training from scratch, pass a model type from the list: """ + """, """.join(snake_case							)}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={
               """help""": (
                   """Override some existing default config settings when a model is trained from scratch. Example: """
                   """n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"""
               )
           }			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}							)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}							)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."""}			,	)
       lowerCAmelCase__						 =  field(
           default="""main"""			,	metadata={"""help""": """The specific model version to use (can be a branch name, tag name or commit id)."""}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={
               """help""": (
                   """Will use the token generated when running `huggingface-cli login` (necessary to use this script """
                   """with private models)."""
               )
           }			,	)






       def 							__A (   self     ):
              if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
                     raise ValueError(
                         '--config_overrides can\'t be used in combination with --config_name or --model_name_or_path'     )



@dataclass
class 	lowerCAmelCase					:
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}							)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}							)
       lowerCAmelCase__						 =  field(default=snake_case			,	metadata={"""help""": """The input training data file (a text file)."""}							)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """An optional input evaluation data file to evaluate the perplexity on (a text file)."""}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """An optional input train ref data file for whole word masking in Chinese."""}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """An optional input validation ref data file for whole word masking in Chinese."""}			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """Overwrite the cached training and evaluation sets"""}							)
       lowerCAmelCase__						 =  field(
           default=5			,	metadata={
               """help""": """The percentage of the train set used as validation set in case there's no validation split"""
           }			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={
               """help""": (
                   """The maximum total input sequence length after tokenization. Sequences longer """
                   """than this will be truncated. Default to the max input length of the model."""
               )
           }			,	)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={"""help""": """The number of processes to use for the preprocessing."""}			,	)
       lowerCAmelCase__						 =  field(
           default=0.15			,	metadata={"""help""": """Ratio of tokens to mask for masked language modeling loss"""}							)
       lowerCAmelCase__						 =  field(
           default=snake_case			,	metadata={
               """help""": (
                   """Whether to pad all samples to `max_seq_length`. """
                   """If False, will pad the samples dynamically when batching to the maximum length in the batch."""
               )
           }			,	)






       def 							__A (   self     ):
              if self.train_file is not None:
                     _UpperCAmelCase     =     self.train_file.split('.'     )[-1]
                     assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
              if self.validation_file is not None:
                     _UpperCAmelCase     =     self.validation_file.split('.'     )[-1]
                     assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."


def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       Optional[int]:






       """simple docstring"""




       with open(SCREAMING_SNAKE_CASE,'r',encoding='utf-8'							) as f:
              _UpperCAmelCase     =     [json.loads(SCREAMING_SNAKE_CASE							) for line in f.read().splitlines() if (len(SCREAMING_SNAKE_CASE							) > 0 and not line.isspace())]
       assert len(SCREAMING_SNAKE_CASE							) == len(SCREAMING_SNAKE_CASE							)

       _UpperCAmelCase     =     {c: dataset[c] for c in dataset.column_names}
       _UpperCAmelCase     =     refs
       return Dataset.from_dict(SCREAMING_SNAKE_CASE							)

def       __lowerCamelCase    (       )		->       Any:






       """simple docstring"""




       _UpperCAmelCase     =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)
       if len(sys.argv							) == 2 and sys.argv[1].endswith('.json'							):
              # If we pass only one argument to the script and it's the path to a json file,
              # let's parse it to get our arguments.
              _UpperCAmelCase    ,							_UpperCAmelCase    ,							_UpperCAmelCase     =     parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]							)							)
       else:
              _UpperCAmelCase    ,							_UpperCAmelCase    ,							_UpperCAmelCase     =     parser.parse_args_into_dataclasses()

       # Detecting last checkpoint.
       _UpperCAmelCase     =     None
       if os.path.isdir(training_args.output_dir							) and training_args.do_train and not training_args.overwrite_output_dir:
              _UpperCAmelCase     =     get_last_checkpoint(training_args.output_dir							)
              if last_checkpoint is None and len(os.listdir(training_args.output_dir							)							) > 0:
                     raise ValueError(
                         F"""Output directory ({training_args.output_dir}) already exists and is not empty. """
                         'Use --overwrite_output_dir to overcome.'							)
              elif last_checkpoint is not None:
                     logger.info(
                         F"""Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change """
                         'the `--output_dir` or add `--overwrite_output_dir` to train from scratch.'							)

    # Setup logging
       logging.basicConfig(
           format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',datefmt='%m/%d/%Y %H:%M:%S',handlers=[logging.StreamHandler(sys.stdout							)],)
       logger.setLevel(logging.INFO if is_main_process(training_args.local_rank							) else logging.WARN							)

       # Log on each process the small summary:
       logger.warning(
           F"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"""
           + F"""distributed training: {bool(training_args.local_rank != -1							)}, 16-bits training: {training_args.fpaa}"""							)
       # Set the verbosity to info of the Transformers logger (on main process only):
       if is_main_process(training_args.local_rank							):
              transformers.utils.logging.set_verbosity_info()
              transformers.utils.logging.enable_default_handler()
              transformers.utils.logging.enable_explicit_format()
       logger.info('Training/evaluation parameters %s',SCREAMING_SNAKE_CASE							)

       # Set seed before initializing model.
       set_seed(training_args.seed							)

       # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
       # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
       # (the dataset will be downloaded automatically from the datasets Hub).
       #
       # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
       # 'text' is found. You can easily tweak this behavior (see below).
       #
       # In distributed training, the load_dataset function guarantee that only one local process can concurrently
       # download the dataset.
       if data_args.dataset_name is not None:
              # Downloading and loading a dataset from the hub.
              _UpperCAmelCase     =     load_dataset(data_args.dataset_name,data_args.dataset_config_name							)
              if "validation" not in datasets.keys():
                     _UpperCAmelCase     =     load_dataset(
                         data_args.dataset_name,data_args.dataset_config_name,split=F"""train[:{data_args.validation_split_percentage}%]""",)
                     _UpperCAmelCase     =     load_dataset(
                         data_args.dataset_name,data_args.dataset_config_name,split=F"""train[{data_args.validation_split_percentage}%:]""",)
       else:
              _UpperCAmelCase     =     {}
              if data_args.train_file is not None:
                     _UpperCAmelCase     =     data_args.train_file
              if data_args.validation_file is not None:
                     _UpperCAmelCase     =     data_args.validation_file
              _UpperCAmelCase     =     data_args.train_file.split('.'							)[-1]
              if extension == "txt":
                     _UpperCAmelCase     =     'text'
              _UpperCAmelCase     =     load_dataset(SCREAMING_SNAKE_CASE,data_files=SCREAMING_SNAKE_CASE							)
       # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
       # https://huggingface.co/docs/datasets/loading_datasets.html.

       # Load pretrained model and tokenizer
       #
       # Distributed training:
       # The .from_pretrained methods guarantee that only one local process can concurrently
       # download model & vocab.
       _UpperCAmelCase     =     {
           'cache_dir': model_args.cache_dir,
           'revision': model_args.model_revision,
           'use_auth_token': True if model_args.use_auth_token else None,
       }
       if model_args.config_name:
              _UpperCAmelCase     =     AutoConfig.from_pretrained(model_args.config_name,**SCREAMING_SNAKE_CASE							)
       elif model_args.model_name_or_path:
              _UpperCAmelCase     =     AutoConfig.from_pretrained(model_args.model_name_or_path,**SCREAMING_SNAKE_CASE							)
       else:
              _UpperCAmelCase     =     CONFIG_MAPPING[model_args.model_type]()
              logger.warning('You are instantiating a new config instance from scratch.'							)
              if model_args.config_overrides is not None:
                     logger.info(F"""Overriding config: {model_args.config_overrides}"""							)
                     config.update_from_string(model_args.config_overrides							)
                     logger.info(F"""New config: {config}"""							)

       _UpperCAmelCase     =     {
           'cache_dir': model_args.cache_dir,
           'use_fast': model_args.use_fast_tokenizer,
           'revision': model_args.model_revision,
           'use_auth_token': True if model_args.use_auth_token else None,
       }
       if model_args.tokenizer_name:
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(model_args.tokenizer_name,**SCREAMING_SNAKE_CASE							)
       elif model_args.model_name_or_path:
              _UpperCAmelCase     =     AutoTokenizer.from_pretrained(model_args.model_name_or_path,**SCREAMING_SNAKE_CASE							)
       else:
              raise ValueError(
                  'You are instantiating a new tokenizer from scratch. This is not supported by this script.'
                  'You can do it from another script, save it, and load it from here, using --tokenizer_name.'							)

       if model_args.model_name_or_path:
              _UpperCAmelCase     =     AutoModelForMaskedLM.from_pretrained(
                  model_args.model_name_or_path,from_tf=bool('.ckpt' in model_args.model_name_or_path							),config=SCREAMING_SNAKE_CASE,cache_dir=model_args.cache_dir,revision=model_args.model_revision,use_auth_token=True if model_args.use_auth_token else None,)
       else:
              logger.info('Training new model from scratch'							)
              _UpperCAmelCase     =     AutoModelForMaskedLM.from_config(SCREAMING_SNAKE_CASE							)

       model.resize_token_embeddings(len(SCREAMING_SNAKE_CASE							)							)

       # Preprocessing the datasets.
       # First we tokenize all the texts.
       if training_args.do_train:
              _UpperCAmelCase     =     datasets['train'].column_names
       else:
              _UpperCAmelCase     =     datasets['validation'].column_names
       _UpperCAmelCase     =     'text' if 'text' in column_names else column_names[0]

       _UpperCAmelCase     =     'max_length' if data_args.pad_to_max_length else False

       def tokenize_function(SCREAMING_SNAKE_CASE							):
              # Remove empty lines
              _UpperCAmelCase     =     [line for line in examples['text'] if len(SCREAMING_SNAKE_CASE							) > 0 and not line.isspace()]
              return tokenizer(examples['text'],padding=SCREAMING_SNAKE_CASE,truncation=SCREAMING_SNAKE_CASE,max_length=data_args.max_seq_length							)

       _UpperCAmelCase     =     datasets.map(
           SCREAMING_SNAKE_CASE,batched=SCREAMING_SNAKE_CASE,num_proc=data_args.preprocessing_num_workers,remove_columns=[text_column_name],load_from_cache_file=not data_args.overwrite_cache,)

       # Add the chinese references if provided
       if data_args.train_ref_file is not None:
              _UpperCAmelCase     =     add_chinese_references(tokenized_datasets['train'],data_args.train_ref_file							)
       if data_args.validation_ref_file is not None:
              _UpperCAmelCase     =     add_chinese_references(
                  tokenized_datasets['validation'],data_args.validation_ref_file							)
       # If we have ref files, need to avoid it removed by trainer
       _UpperCAmelCase     =     data_args.train_ref_file or data_args.validation_ref_file
       if has_ref:
              _UpperCAmelCase     =     False

       # Data collator
       # This one will take care of randomly masking the tokens.
       _UpperCAmelCase     =     DataCollatorForWholeWordMask(tokenizer=SCREAMING_SNAKE_CASE,mlm_probability=data_args.mlm_probability							)

       # Initialize our Trainer
       _UpperCAmelCase     =     Trainer(
           model=SCREAMING_SNAKE_CASE,args=SCREAMING_SNAKE_CASE,train_dataset=tokenized_datasets['train'] if training_args.do_train else None,eval_dataset=tokenized_datasets['validation'] if training_args.do_eval else None,tokenizer=SCREAMING_SNAKE_CASE,data_collator=SCREAMING_SNAKE_CASE,)

       # Training
       if training_args.do_train:
              if last_checkpoint is not None:
                     _UpperCAmelCase     =     last_checkpoint
              elif model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path							):
                     _UpperCAmelCase     =     model_args.model_name_or_path
              else:
                     _UpperCAmelCase     =     None
              _UpperCAmelCase     =     trainer.train(resume_from_checkpoint=SCREAMING_SNAKE_CASE							)
              trainer.save_model()  # Saves the tokenizer too for easy upload

              _UpperCAmelCase     =     os.path.join(training_args.output_dir,'train_results.txt'							)
              if trainer.is_world_process_zero():
                     with open(SCREAMING_SNAKE_CASE,'w'							) as writer:
                            logger.info('***** Train results *****'							)
                            for key, value in sorted(train_result.metrics.items()							):
                                   logger.info(F"""  {key} = {value}"""							)
                                   writer.write(F"""{key} = {value}\n"""							)

            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
                     trainer.state.save_to_json(os.path.join(training_args.output_dir,'trainer_state.json'							)							)

    # Evaluation
       _UpperCAmelCase     =     {}
       if training_args.do_eval:
              logger.info('*** Evaluate ***'							)

              _UpperCAmelCase     =     trainer.evaluate()

              _UpperCAmelCase     =     math.exp(eval_output['eval_loss']							)
              _UpperCAmelCase     =     perplexity

              _UpperCAmelCase     =     os.path.join(training_args.output_dir,'eval_results_mlm_wwm.txt'							)
              if trainer.is_world_process_zero():
                     with open(SCREAMING_SNAKE_CASE,'w'							) as writer:
                            logger.info('***** Eval results *****'							)
                            for key, value in sorted(results.items()							):
                                   logger.info(F"""  {key} = {value}"""							)
                                   writer.write(F"""{key} = {value}\n"""							)

       return results

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE							)		->       Any:






       """simple docstring"""




       main()


if __name__ == "__main__":
    main()













"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_			=							logging.get_logger(__name__)

lowerCAmelCase_			=							'''▁'''

lowerCAmelCase_			=							{'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

lowerCAmelCase_			=							{
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

lowerCAmelCase_			=							{'''vinai/bartpho-syllable''': 1_024}



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  VOCAB_FILES_NAMES
       lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__						 =  ["""input_ids""", """attention_mask"""]






       def __init__(   self    ,  a__    ,  a__    ,  a__="<s>"    ,  a__="</s>"    ,  a__="</s>"    ,  a__="<s>"    ,  a__="<unk>"    ,  a__="<pad>"    ,  a__="<mask>"    ,  a__ = None    ,  **a__    ,  ):
              # Mask token behave like a normal word, i.e. include the space before it
              _UpperCAmelCase     =     AddedToken(a__    ,  lstrip=a__    ,  rstrip=a__     ) if isinstance(a__    ,  a__     ) else mask_token

              _UpperCAmelCase     =     {} if sp_model_kwargs is None else sp_model_kwargs

              super().__init__(
                  bos_token=a__    ,  eos_token=a__    ,  unk_token=a__    ,  sep_token=a__    ,  cls_token=a__    ,  pad_token=a__    ,  mask_token=a__    ,  sp_model_kwargs=self.sp_model_kwargs    ,  **a__    ,  )

              _UpperCAmelCase     =     vocab_file
              _UpperCAmelCase     =     monolingual_vocab_file
              _UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
              self.sp_model.Load(str(a__     )     )

              # Load the reduced vocab

              # Keep order of special tokens for backward compatibility
              _UpperCAmelCase     =     {}
              _UpperCAmelCase     =     0
              for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
                     if str(a__     ) not in self.fairseq_tokens_to_ids:
                            _UpperCAmelCase     =     cnt
                            cnt += 1
              with open(a__    ,  'r'    ,  encoding='utf-8'     ) as f:
                     for line in f.readlines():
                            _UpperCAmelCase     =     line.strip().split()[0]
                            _UpperCAmelCase     =     len(self.fairseq_tokens_to_ids     )
              if str(a__     ) not in self.fairseq_tokens_to_ids:
                     _UpperCAmelCase     =     len(self.fairseq_tokens_to_ids     )

              _UpperCAmelCase     =     {v: k for k, v in self.fairseq_tokens_to_ids.items()}






       def __getstate__(   self     ):
              _UpperCAmelCase     =     self.__dict__.copy()
              _UpperCAmelCase     =     None
              _UpperCAmelCase     =     self.sp_model.serialized_model_proto()
              return state






       def __setstate__(   self    ,  a__     ):
              _UpperCAmelCase     =     d

              # for backward compatibility
              if not hasattr(self    ,  'sp_model_kwargs'     ):
                     _UpperCAmelCase     =     {}

              _UpperCAmelCase     =     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
              self.sp_model.LoadFromSerializedProto(self.sp_model_proto     )






       def 							__A (   self    ,  a__    ,  a__ = None     ):

              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              _UpperCAmelCase     =     [self.cls_token_id]
              _UpperCAmelCase     =     [self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep






       def 							__A (   self    ,  a__    ,  a__ = None    ,  a__ = False     ):

              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=a__    ,  token_ids_a=a__    ,  already_has_special_tokens=a__     )

              if token_ids_a is None:
                     return [1] + ([0] * len(a__     )) + [1]
              return [1] + ([0] * len(a__     )) + [1, 1] + ([0] * len(a__     )) + [1]






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              _UpperCAmelCase     =     [self.sep_token_id]
              _UpperCAmelCase     =     [self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep     ) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]






       @property
       def 							__A (   self     ):
              return len(self.fairseq_ids_to_tokens     )






       def 							__A (   self     ):
              _UpperCAmelCase     =     {self.convert_ids_to_tokens(a__     ): i for i in range(self.vocab_size     )}
              vocab.update(self.added_tokens_encoder     )
              return vocab






       def 							__A (   self    ,  a__     ):
              return self.sp_model.encode(a__    ,  out_type=a__     )






       def 							__A (   self    ,  a__     ):
              if token in self.fairseq_tokens_to_ids:
                     return self.fairseq_tokens_to_ids[token]
              else:
                     return self.unk_token_id






       def 							__A (   self    ,  a__     ):
              return self.fairseq_ids_to_tokens[index]






       def 							__A (   self    ,  a__     ):
              _UpperCAmelCase     =     ''.join(a__     ).replace(a__    ,  ' '     ).strip()
              return out_string






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              if not os.path.isdir(a__     ):
                     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
                     return
              _UpperCAmelCase     =     os.path.join(
                  a__    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
              _UpperCAmelCase     =     os.path.join(
                  a__    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file']    ,  )

              if os.path.abspath(self.vocab_file     ) != os.path.abspath(a__     ) and os.path.isfile(self.vocab_file     ):
                     copyfile(self.vocab_file    ,  a__     )
              elif not os.path.isfile(self.vocab_file     ):
                     with open(a__    ,  'wb'     ) as fi:
                            _UpperCAmelCase     =     self.sp_model.serialized_model_proto()
                            fi.write(a__     )

              if os.path.abspath(self.monolingual_vocab_file     ) != os.path.abspath(
                  a__     ) and os.path.isfile(self.monolingual_vocab_file     ):
                     copyfile(self.monolingual_vocab_file    ,  a__     )
              elif not os.path.isfile(self.monolingual_vocab_file     ):
                     with open(a__    ,  'w'    ,  encoding='utf-8'     ) as fp:
                            for token in self.fairseq_tokens_to_ids:
                                   if token not in self.all_special_tokens:
                                          fp.write(f"""{str(a__     )} \n"""     )

              return out_vocab_file, out_monolingual_vocab_file






