
import logging
import os
import threading
import time


try:
							import warnings
except ImportError:
							_UpperCAmelCase      :				Tuple          = None

try:
							import msvcrt
except ImportError:
							_UpperCAmelCase      :				List[str]          = None

try:
							import fcntl
except ImportError:
							_UpperCAmelCase      :				int          = None


# Backward compatibility
# ------------------------------------------------
try:
							TimeoutError
except NameError:
							_UpperCAmelCase      :				Optional[int]          = OSError


# Data
# ------------------------------------------------
_UpperCAmelCase      :				int          = [
    "Timeout",
    "BaseFileLock",
    "WindowsFileLock",
    "UnixFileLock",
    "SoftFileLock",
    "FileLock",
]

_UpperCAmelCase      :				List[str]          = "3.0.12"


_UpperCAmelCase      :				str          = None


def    A (      )     ->						Dict:
		'''simple docstring'''

		global _logger
		UpperCamelCase						=  _logger or logging.getLogger(__name__				)
		return _logger


class        lowercase      (	_SCREAMING_SNAKE_CASE     ):



		def __init__(				self       ,   A_ )						->      Tuple:
				"""simple docstring"""
				UpperCamelCase						=  lock_file
				return None



		def __str__(				self )						->      Any:
				"""simple docstring"""
				UpperCamelCase						=  F'''The file lock \'{self.lock_file}\' could not be acquired.'''
				return temp






class        lowercase      :



		def __init__(				self       ,   A_ )						->      str:
				"""simple docstring"""
				UpperCamelCase						=  lock
				return None



		def __enter__(				self )						->      Optional[int]:
				"""simple docstring"""
				return self.lock



		def __exit__(				self       ,   A_       ,   A_       ,   A_ )						->      List[str]:
				"""simple docstring"""
				self.lock.release()
				return None






class        lowercase      :



		def __init__(				self       ,   A_       ,   A_=-1       ,   A_=None )						->      int:
				"""simple docstring"""
				UpperCamelCase						=  max_filename_length if max_filename_length is not None else 255
				# Hash the filename if it's too long
				UpperCamelCase						=  self.hash_filename_if_too_long(A_       ,   A_ )
				# The path to the lock file.
				UpperCamelCase						=  lock_file

				# The file descriptor for the *_lock_file* as it is returned by the
				# os.open() function.
				# This file lock is only NOT None, if the object currently holds the
				# lock.
				UpperCamelCase						=  None

				# The default timeout value.
				UpperCamelCase						=  timeout

				# We use this lock primarily for the lock counter.
				UpperCamelCase						=  threading.Lock()

				# The lock counter is used for implementing the nested locking
				# mechanism. Whenever the lock is acquired, the counter is increased and
				# the lock is only released, when this value is 0 again.
				UpperCamelCase						=  0
				return None



		@property
		def __UpperCamelCase				(				self )						->      str:
				"""simple docstring"""
				return self._lock_file



		@property
		def __UpperCamelCase				(				self )						->      List[str]:
				"""simple docstring"""
				return self._timeout



		@timeout.setter
		def __UpperCamelCase				(				self       ,   A_ )						->      int:
				"""simple docstring"""
				UpperCamelCase						=  float(A_ )
				return None



		def __UpperCamelCase				(				self )						->      str:
				"""simple docstring"""
				raise NotImplementedError()



		def __UpperCamelCase				(				self )						->      Union[str, Any]:
				"""simple docstring"""
				raise NotImplementedError()



		@property
		def __UpperCamelCase				(				self )						->      List[str]:
				"""simple docstring"""
				return self._lock_file_fd is not None



		def __UpperCamelCase				(				self       ,   A_=None       ,   A_=0.05 )						->      List[str]:
				"""simple docstring"""
				# Use the default timeout, if no timeout is provided.
				if timeout is None:
						UpperCamelCase						=  self.timeout

				# Increment the number right at the beginning.
				# We can still undo it, if something fails.
				with self._thread_lock:
						self._lock_counter += 1

				UpperCamelCase						=  id(self )
				UpperCamelCase						=  self._lock_file
				UpperCamelCase						=  time.time()
				try:
						while True:
								with self._thread_lock:
										if not self.is_locked:
												logger().debug(F'''Attempting to acquire lock {lock_id} on {lock_filename}''' )
												self._acquire()

								if self.is_locked:
										logger().debug(F'''Lock {lock_id} acquired on {lock_filename}''' )
										break
								elif timeout >= 0 and time.time() - start_time > timeout:
										logger().debug(F'''Timeout on acquiring lock {lock_id} on {lock_filename}''' )
										raise Timeout(self._lock_file )
								else:
										logger().debug(
										    F'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...''' )
										time.sleep(A_ )
				except:  # noqa
						# Something did go wrong, so decrement the counter.
						with self._thread_lock:
								UpperCamelCase						=  max(0       ,   self._lock_counter - 1 )

						raise
				return _Acquire_ReturnProxy(lock=self )



		def __UpperCamelCase				(				self       ,   A_=False )						->      Any:
				"""simple docstring"""
				with self._thread_lock:
						if self.is_locked:
								self._lock_counter -= 1

								if self._lock_counter == 0 or force:
										UpperCamelCase						=  id(self )
										UpperCamelCase						=  self._lock_file

										logger().debug(F'''Attempting to release lock {lock_id} on {lock_filename}''' )
										self._release()
										UpperCamelCase						=  0
										logger().debug(F'''Lock {lock_id} released on {lock_filename}''' )

				return None



		def __enter__(				self )						->      List[str]:
				"""simple docstring"""
				self.acquire()
				return self



		def __exit__(				self       ,   A_       ,   A_       ,   A_ )						->      Optional[int]:
				"""simple docstring"""
				self.release()
				return None



		def __del__(				self )						->      str:
				"""simple docstring"""
				self.release(force=A_ )
				return None



		def __UpperCamelCase				(				self       ,   A_       ,   A_ )						->      str:
				"""simple docstring"""
				UpperCamelCase						=  os.path.basename(A_ )
				if len(A_ ) > max_length and max_length > 0:
						UpperCamelCase						=  os.path.dirname(A_ )
						UpperCamelCase						=  str(hash(A_ ) )
						UpperCamelCase						=  filename[: max_length - len(A_ ) - 8] + '...' + hashed_filename + '.lock'
						return os.path.join(A_       ,   A_ )
				else:
						return path






class        lowercase      (	_SCREAMING_SNAKE_CASE     ):



		def __init__(				self       ,   A_       ,   A_=-1       ,   A_=None )						->      List[str]:
				"""simple docstring"""
				from .file_utils import relative_to_absolute_path

				super().__init__(A_       ,   timeout=A_       ,   max_filename_length=A_ )
				UpperCamelCase						=  '\\\\?\\' + relative_to_absolute_path(self.lock_file )



		def __UpperCamelCase				(				self )						->      int:
				"""simple docstring"""
				UpperCamelCase						=  os.O_RDWR | os.O_CREAT | os.O_TRUNC

				try:
						UpperCamelCase						=  os.open(self._lock_file       ,   A_ )
				except OSError:
						pass
				else:
						try:
								msvcrt.locking(A_       ,   msvcrt.LK_NBLCK       ,   1 )
						except OSError:
								os.close(A_ )
						else:
								UpperCamelCase						=  fd
				return None



		def __UpperCamelCase				(				self )						->      List[str]:
				"""simple docstring"""
				UpperCamelCase						=  self._lock_file_fd
				UpperCamelCase						=  None
				msvcrt.locking(A_       ,   msvcrt.LK_UNLCK       ,   1 )
				os.close(A_ )

				try:
						os.remove(self._lock_file )
				# Probably another instance of the application
				# that acquired the file lock.
				except OSError:
						pass
				return None






class        lowercase      (	_SCREAMING_SNAKE_CASE     ):



		def __init__(				self       ,   A_       ,   A_=-1       ,   A_=None )						->      Optional[int]:
				"""simple docstring"""
				UpperCamelCase						=  os.statvfs(os.path.dirname(A_ ) ).f_namemax
				super().__init__(A_       ,   timeout=A_       ,   max_filename_length=A_ )



		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				UpperCamelCase						=  os.O_RDWR | os.O_CREAT | os.O_TRUNC
				UpperCamelCase						=  os.open(self._lock_file       ,   A_ )

				try:
						fcntl.flock(A_       ,   fcntl.LOCK_EX | fcntl.LOCK_NB )
				except OSError:
						os.close(A_ )
				else:
						UpperCamelCase						=  fd
				return None



		def __UpperCamelCase				(				self )						->      Optional[Any]:
				"""simple docstring"""
				UpperCamelCase						=  self._lock_file_fd
				UpperCamelCase						=  None
				fcntl.flock(A_       ,   fcntl.LOCK_UN )
				os.close(A_ )
				return None






class        lowercase      (	_SCREAMING_SNAKE_CASE     ):



		def __UpperCamelCase				(				self )						->      str:
				"""simple docstring"""
				UpperCamelCase						=  os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
				try:
						UpperCamelCase						=  os.open(self._lock_file       ,   A_ )
				except OSError:
						pass
				else:
						UpperCamelCase						=  fd
				return None



		def __UpperCamelCase				(				self )						->      List[str]:
				"""simple docstring"""
				os.close(self._lock_file_fd )
				UpperCamelCase						=  None

				try:
						os.remove(self._lock_file )
				# The file is already deleted and that's what we want.
				except OSError:
						pass
				return None






_UpperCAmelCase      :				str          = None

if msvcrt:
							_UpperCAmelCase      :				Tuple          = WindowsFileLock
elif fcntl:
							_UpperCAmelCase      :				Any          = UnixFileLock
else:
							_UpperCAmelCase      :				int          = SoftFileLock

							if warnings is not None:
														warnings.warn("only soft file lock is available")


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast


class        lowercase      (	unittest.TestCase     ):



		def __UpperCamelCase				(				self )						->      Tuple:
				"""simple docstring"""
				# A mock response for an HTTP head request to emulate server down
				UpperCamelCase						=  mock.Mock()
				UpperCamelCase						=  500
				UpperCamelCase						=  {}
				UpperCamelCase						=  HTTPError
				UpperCamelCase						=  {}

				# Download this model to make sure it's in the cache.
				UpperCamelCase						=  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('requests.Session.request'       ,   return_value=A_ ) as mock_head:
						UpperCamelCase						=  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )
						# This check we did call the fake head request
						mock_head.assert_called()



		@require_tokenizers
		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				# A mock response for an HTTP head request to emulate server down
				UpperCamelCase						=  mock.Mock()
				UpperCamelCase						=  500
				UpperCamelCase						=  {}
				UpperCamelCase						=  HTTPError
				UpperCamelCase						=  {}

				# Download this model to make sure it's in the cache.
				UpperCamelCase						=  GPTaTokenizerFast.from_pretrained('gpt2' )

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('requests.Session.request'       ,   return_value=A_ ) as mock_head:
						UpperCamelCase						=  GPTaTokenizerFast.from_pretrained('gpt2' )
						# This check we did call the fake head request
						mock_head.assert_called()



		def __UpperCamelCase				(				self )						->      Optional[Any]:
				"""simple docstring"""
				# This test is for deprecated behavior and can be removed in v5
				try:
						UpperCamelCase						=  tempfile.mktemp()
						with open(A_       ,   'wb' ) as f:
								http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       ,   A_ )

						UpperCamelCase						=  AlbertTokenizer.from_pretrained(A_ )
				finally:
						os.remove(A_ )

				# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
				# the current folder and have the right name.
				if os.path.isfile('tokenizer.json' ):
						# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
						return
				try:
						with open('tokenizer.json'       ,   'wb' ) as f:
								http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'       ,   A_ )
						UpperCamelCase						=  AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2' )
						# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
						self.assertEqual(tokenizer.vocab_size       ,   1_000 )
						# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

				finally:
						os.remove('tokenizer.json' )



		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				# This test is for deprecated behavior and can be removed in v5
				UpperCamelCase						=  AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model' )






@is_staging_test
class        lowercase      (	unittest.TestCase     ):
		__lowercase : int			  =       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]



		@classmethod
		def __UpperCamelCase				(				cls )						->      Tuple:
				"""simple docstring"""
				UpperCamelCase						=  TOKEN
				HfFolder.save_token(A_ )



		@classmethod
		def __UpperCamelCase				(				cls )						->      Optional[int]:
				"""simple docstring"""
				try:
						delete_repo(token=cls._token       ,   repo_id='test-tokenizer' )
				except HTTPError:
						pass

				try:
						delete_repo(token=cls._token       ,   repo_id='valid_org/test-tokenizer-org' )
				except HTTPError:
						pass

				try:
						delete_repo(token=cls._token       ,   repo_id='test-dynamic-tokenizer' )
				except HTTPError:
						pass



		def __UpperCamelCase				(				self )						->      Any:
				"""simple docstring"""
				with tempfile.TemporaryDirectory() as tmp_dir:
						UpperCamelCase						=  os.path.join(A_       ,   'vocab.txt' )
						with open(A_       ,   'w'       ,   encoding='utf-8' ) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
						UpperCamelCase						=  BertTokenizer(A_ )

				tokenizer.push_to_hub('test-tokenizer'       ,   use_auth_token=self._token )
				UpperCamelCase						=  BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''' )
				self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab )

				# Reset repo
				delete_repo(token=self._token       ,   repo_id='test-tokenizer' )

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(A_       ,   repo_id='test-tokenizer'       ,   push_to_hub=A_       ,   use_auth_token=self._token )

				UpperCamelCase						=  BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer''' )
				self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab )



		def __UpperCamelCase				(				self )						->      int:
				"""simple docstring"""
				with tempfile.TemporaryDirectory() as tmp_dir:
						UpperCamelCase						=  os.path.join(A_       ,   'vocab.txt' )
						with open(A_       ,   'w'       ,   encoding='utf-8' ) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
						UpperCamelCase						=  BertTokenizer(A_ )

				tokenizer.push_to_hub('valid_org/test-tokenizer-org'       ,   use_auth_token=self._token )
				UpperCamelCase						=  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
				self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab )

				# Reset repo
				delete_repo(token=self._token       ,   repo_id='valid_org/test-tokenizer-org' )

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(
						    A_       ,   repo_id='valid_org/test-tokenizer-org'       ,   push_to_hub=A_       ,   use_auth_token=self._token )

				UpperCamelCase						=  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
				self.assertDictEqual(new_tokenizer.vocab       ,   tokenizer.vocab )



		@require_tokenizers
		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				CustomTokenizer.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
						UpperCamelCase						=  os.path.join(A_       ,   'vocab.txt' )
						with open(A_       ,   'w'       ,   encoding='utf-8' ) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
						UpperCamelCase						=  CustomTokenizer(A_ )

				# No fast custom tokenizer
				tokenizer.push_to_hub('test-dynamic-tokenizer'       ,   use_auth_token=self._token )

				UpperCamelCase						=  AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''       ,   trust_remote_code=A_ )
				# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__       ,   'CustomTokenizer' )

				# Fast and slow custom tokenizer
				CustomTokenizerFast.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
						UpperCamelCase						=  os.path.join(A_       ,   'vocab.txt' )
						with open(A_       ,   'w'       ,   encoding='utf-8' ) as vocab_writer:
								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )

						UpperCamelCase						=  BertTokenizerFast.from_pretrained(A_ )
						bert_tokenizer.save_pretrained(A_ )
						UpperCamelCase						=  CustomTokenizerFast.from_pretrained(A_ )

				tokenizer.push_to_hub('test-dynamic-tokenizer'       ,   use_auth_token=self._token )

				UpperCamelCase						=  AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''       ,   trust_remote_code=A_ )
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__       ,   'CustomTokenizerFast' )
				UpperCamelCase						=  AutoTokenizer.from_pretrained(
				    F'''{USER}/test-dynamic-tokenizer'''       ,   use_fast=A_       ,   trust_remote_code=A_ )
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__       ,   'CustomTokenizer' )






class        lowercase      (	unittest.TestCase     ):



		def __UpperCamelCase				(				self )						->      Optional[int]:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('Hello 友達' )
				self.assertEqual(trie.data       ,   {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}} )
				trie.add('Hello' )
				trie.data
				self.assertEqual(trie.data       ,   {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}} )



		def __UpperCamelCase				(				self )						->      str:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				self.assertEqual(trie.split('[CLS] This is a extra_id_100' )       ,   ['[CLS] This is a extra_id_100'] )
				trie.add('[CLS]' )
				trie.add('extra_id_1' )
				trie.add('extra_id_100' )
				self.assertEqual(trie.split('[CLS] This is a extra_id_100' )       ,   ['[CLS]', ' This is a ', 'extra_id_100'] )



		def __UpperCamelCase				(				self )						->      List[Any]:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('A' )
				self.assertEqual(trie.split('ABC' )       ,   ['A', 'BC'] )
				self.assertEqual(trie.split('BCA' )       ,   ['BC', 'A'] )



		def __UpperCamelCase				(				self )						->      Tuple:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('TOKEN]' )
				trie.add('[SPECIAL_TOKEN]' )
				self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )       ,   ['This is something ', '[SPECIAL_TOKEN]'] )



		def __UpperCamelCase				(				self )						->      Dict:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('A' )
				trie.add('P' )
				trie.add('[SPECIAL_TOKEN]' )
				self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )       ,   ['This is something ', '[SPECIAL_TOKEN]'] )



		def __UpperCamelCase				(				self )						->      Optional[Any]:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('AB' )
				trie.add('B' )
				trie.add('C' )
				self.assertEqual(trie.split('ABC' )       ,   ['AB', 'C'] )



		def __UpperCamelCase				(				self )						->      Tuple:
				"""simple docstring"""
				UpperCamelCase						=  Trie()
				trie.add('ABC' )
				trie.add('B' )
				trie.add('CD' )
				self.assertEqual(trie.split('ABCD' )       ,   ['ABC', 'D'] )



		def __UpperCamelCase				(				self )						->      int:
				"""simple docstring"""
				# Even if the offsets are wrong, we necessarily output correct string
				# parts.
				UpperCamelCase						=  Trie()
				UpperCamelCase						=  trie.cut_text('ABC'       ,   [0, 0, 2, 1, 2, 3] )
				self.assertEqual(A_       ,   ['AB', 'C'] )
