
"""simple docstring"""





import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s''',
    datefmt='''%m/%d/%Y %H:%M:%S''',
    level=logging.INFO,
)
UpperCamelCase_					:						Tuple				= logging.getLogger(__name__)
def 							A_     (__a				):

					'''simple docstring'''





					A_				 =						git.Repo(search_parent_directories=__a				)
					A_				 =						{
					    "repo_id": str(__a				),
					    "repo_sha": str(repo.head.object.hexsha				),
					    "repo_branch": str(repo.active_branch				),
					}

					with open(os.path.join(__a				,   "git_log.json"				)				,   "w"				) as f:
										json.dump(__a				,   __a				,   indent=4				)
def 							A_     (__a				):

					'''simple docstring'''





					if params.n_gpu <= 0:
										A_				 =						0
										A_				 =						-1
										A_				 =						True
										A_				 =						False
										return

					assert torch.cuda.is_available()

					logger.info("Initializing GPUs"				)
					if params.n_gpu > 1:
										assert params.local_rank != -1

										A_				 =						int(os.environ["WORLD_SIZE"]				)
										A_				 =						int(os.environ["N_GPU_NODE"]				)
										A_				 =						int(os.environ["RANK"]				)

										# number of nodes / node ID
										A_				 =						params.world_size // params.n_gpu_per_node
										A_				 =						params.global_rank // params.n_gpu_per_node
										A_				 =						True

										assert params.n_nodes == int(os.environ["N_NODES"]				)
										assert params.node_id == int(os.environ["NODE_RANK"]				)

					# local job (single GPU)
					else:
										assert params.local_rank == -1

										A_				 =						1
										A_				 =						0
										A_				 =						0
										A_				 =						0
										A_				 =						1
										A_				 =						1
										A_				 =						False

					# sanity checks
					assert params.n_nodes >= 1
					assert 0 <= params.node_id < params.n_nodes
					assert 0 <= params.local_rank <= params.global_rank < params.world_size
					assert params.world_size == params.n_nodes * params.n_gpu_per_node

					# define whether this is the master process / if we are in multi-node distributed mode
					A_				 =						params.node_id == 0 and params.local_rank == 0
					A_				 =						params.n_nodes > 1

					# summary
					A_				 =						f'--- Global rank: {params.global_rank} - '
					logger.info(PREFIX + "Number of nodes: %i" % params.n_nodes				)
					logger.info(PREFIX + "Node ID        : %i" % params.node_id				)
					logger.info(PREFIX + "Local rank     : %i" % params.local_rank				)
					logger.info(PREFIX + "World size     : %i" % params.world_size				)
					logger.info(PREFIX + "GPUs per node  : %i" % params.n_gpu_per_node				)
					logger.info(PREFIX + "Master         : %s" % str(params.is_master				)				)
					logger.info(PREFIX + "Multi-node     : %s" % str(params.multi_node				)				)
					logger.info(PREFIX + "Multi-GPU      : %s" % str(params.multi_gpu				)				)
					logger.info(PREFIX + "Hostname       : %s" % socket.gethostname()				)

					# set GPU device
					torch.cuda.set_device(params.local_rank				)

					# initialize multi-GPU
					if params.multi_gpu:
										logger.info("Initializing PyTorch distributed"				)
										torch.distributed.init_process_group(
										    init_method="env://"				,   backend="nccl"				,   )




def 							A_     (__a				):

					'''simple docstring'''





					np.random.seed(args.seed				)
					torch.manual_seed(args.seed				)
					if args.n_gpu > 0:
										torch.cuda.manual_seed_all(args.seed				)






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase_					:						Any				= logging.get_logger(__name__)

UpperCamelCase_					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase_					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)




