


'''simple docstring'''

import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


a_						:       List[Any]      			=				logging.get_logger(__name__)



class   snake_case					:





  """simple docstring"""




  _lowerCamelCase						=						None





@experimental
def __snake_case      (   UpperCAmelCase_		:		Union[str, Any]      ,       UpperCAmelCase_		:		Dict      ,       UpperCAmelCase_		:		Any      ,       UpperCAmelCase_		:		List[str]      ,       UpperCAmelCase_		:		Any      ,       UpperCAmelCase_		:		str      ,       UpperCAmelCase_		:		Any	):
 if ParallelBackendConfig.backend_name is None:
  return _map_with_multiprocessing_pool(
      UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_	)

 return _map_with_joblib(UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_	)





def __snake_case      (   UpperCAmelCase_		:		int      ,       UpperCAmelCase_		:		str      ,       UpperCAmelCase_		:		Tuple      ,       UpperCAmelCase_		:		List[Any]      ,       UpperCAmelCase_		:		List[Any]      ,       UpperCAmelCase_		:		Any      ,       UpperCAmelCase_		:		List[Any]	):
 lowerCamelCase_  				=						num_proc if num_proc <= len(UpperCAmelCase_	) else len(UpperCAmelCase_	)
 lowerCamelCase_  				=						[]  # We organize the splits ourselve (contiguous splits)
 for index in range(UpperCAmelCase_	):
  lowerCamelCase_  				=						len(UpperCAmelCase_	) // num_proc
  lowerCamelCase_  				=						len(UpperCAmelCase_	) % num_proc
  lowerCamelCase_  				=						div * index + min(UpperCAmelCase_      ,       UpperCAmelCase_	)
  lowerCamelCase_  				=						start + div + (1 if index < mod else 0)
  split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)	)

 if len(UpperCAmelCase_	) != sum(len(i[1]	) for i in split_kwds	):
  raise ValueError(
      F'''Error dividing inputs iterable among processes. '''
      F'''Total number of objects {len(UpperCAmelCase_	)}, '''
      F'''length: {sum(len(i[1]	) for i in split_kwds	)}'''	)

 logger.info(
     F'''Spawning {num_proc} processes for {len(UpperCAmelCase_	)} objects in slices of {[len(i[1]	) for i in split_kwds]}'''	)
 lowerCamelCase_				,lowerCamelCase_  				=						None, None
 if not disable_tqdm:
  lowerCamelCase_				,lowerCamelCase_  				=						(RLock(),), tqdm.set_lock
 with Pool(UpperCAmelCase_      ,       initargs=UpperCAmelCase_      ,       initializer=UpperCAmelCase_	) as pool:
  lowerCamelCase_  				=						pool.map(UpperCAmelCase_      ,       UpperCAmelCase_	)
 logger.info(F'''Finished {num_proc} processes'''	)
 lowerCamelCase_  				=						[obj for proc_res in mapped for obj in proc_res]
 logger.info(F'''Unpacked {len(UpperCAmelCase_	)} objects'''	)

 return mapped





def __snake_case      (   UpperCAmelCase_		:		Optional[int]      ,       UpperCAmelCase_		:		str      ,       UpperCAmelCase_		:		str      ,       UpperCAmelCase_		:		List[str]      ,       UpperCAmelCase_		:		Dict      ,       UpperCAmelCase_		:		Optional[Any]      ,       UpperCAmelCase_		:		Dict	):
 # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
 # and it requires monkey-patching joblib internal classes which is subject to change
 import joblib

 with joblib.parallel_backend(ParallelBackendConfig.backend_name      ,       n_jobs=UpperCAmelCase_	):
  return joblib.Parallel()(
      joblib.delayed(UpperCAmelCase_	)((function, obj, types, None, True, None)	) for obj in iterable	)







@experimental
@contextlib.contextmanager
def __snake_case      (   UpperCAmelCase_		:		str	):
 lowerCamelCase_  				=						backend_name

 if backend_name == "spark":
  from joblibspark import register_spark

  register_spark()

  # TODO: call create_cache_and_write_probe if "download" in steps
  # TODO: raise NotImplementedError when Dataset.map etc is called

 try:
  yield
 finally:
  lowerCamelCase_  				=						None







'''simple docstring'''

import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


a_						:       Dict      			=				logging.get_logger(__name__)

a_						:       List[Any]      			=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_						:       Optional[Any]      			=				{
    """tokenizer_file""": {
        """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json""",
    },
}

a_						:       str      			=				{
    """gpt-neox-20b""": 2048,
}





class   snake_case					(					lowercase    ):





  """simple docstring"""




  _lowerCamelCase						=						VOCAB_FILES_NAMES
  _lowerCamelCase						=						PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase						=						["input_ids", "attention_mask"]






  def __init__(					self    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase=None    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase="<|endoftext|>"    ,						UpperCamelCase=False    ,						**UpperCamelCase    ,						):







   """simple docstring"""
   super().__init__(
       UpperCamelCase    ,						UpperCamelCase    ,						tokenizer_file=UpperCamelCase    ,						unk_token=UpperCamelCase    ,						bos_token=UpperCamelCase    ,						eos_token=UpperCamelCase    ,						add_prefix_space=UpperCamelCase    ,						**UpperCamelCase    ,						)

   lowerCamelCase_  				=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
   if pre_tok_state.get("add_prefix_space"    ,						UpperCamelCase  ) != add_prefix_space:
    lowerCamelCase_  				=						getattr(UpperCamelCase    ,						pre_tok_state.pop("type"  )  )
    lowerCamelCase_  				=						add_prefix_space
    lowerCamelCase_  				=						pre_tok_class(**UpperCamelCase  )

   lowerCamelCase_  				=						add_prefix_space






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase = None  ):







   """simple docstring"""
   lowerCamelCase_  				=						self._tokenizer.model.save(UpperCamelCase    ,						name=UpperCamelCase  )
   return tuple(UpperCamelCase  )







  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						[]
   for is_user, text in conversation.iter_texts():
    input_ids.extend(self.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  ) + [self.eos_token_id]  )

   if len(UpperCamelCase  ) > self.model_max_length:
    lowerCamelCase_  				=						input_ids[-self.model_max_length :]
   return input_ids



