
"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast




class 				lowercase__(				unittest.TestCase   ):
	'''simple docstring'''

	def   __lowerCAmelCase (						self    :Dict					)							->			int:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[str]				= mock.Mock()
			SCREAMING_SNAKE_CASE       :     Any				= 5_00
			SCREAMING_SNAKE_CASE       :     int				= {}
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= HTTPError
			SCREAMING_SNAKE_CASE       :     int				= {}

			# Download this model to make sure it's in the cache.
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)

			# Under the mock environment we get a 500 error when trying to reach the tokenizer.
			with mock.patch('''requests.Session.request'''		, return_value=lowerCamelCase_					) as mock_head:
					SCREAMING_SNAKE_CASE       :     List[str]				= BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)
					# This check we did call the fake head request
					mock_head.assert_called()





	@require_tokenizers
	def   __lowerCAmelCase (						self    :List[Any]					)							->			Any:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[str]				= mock.Mock()
			SCREAMING_SNAKE_CASE       :     Any				= 5_00
			SCREAMING_SNAKE_CASE       :     Any				= {}
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= HTTPError
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= {}

			# Download this model to make sure it's in the cache.
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= GPTaTokenizerFast.from_pretrained('''gpt2'''					)

			# Under the mock environment we get a 500 error when trying to reach the tokenizer.
			with mock.patch('''requests.Session.request'''		, return_value=lowerCamelCase_					) as mock_head:
					SCREAMING_SNAKE_CASE       :     Optional[Any]				= GPTaTokenizerFast.from_pretrained('''gpt2'''					)
					# This check we did call the fake head request
					mock_head.assert_called()





	def   __lowerCAmelCase (						self    :Any					)							->			Optional[Any]:







			'''simple docstring'''




			try:
					SCREAMING_SNAKE_CASE       :     Optional[int]				= tempfile.mktemp()
					with open(lowerCamelCase_		, '''wb'''					) as f:
							http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		, lowerCamelCase_					)

					SCREAMING_SNAKE_CASE       :     List[str]				= AlbertTokenizer.from_pretrained(lowerCamelCase_					)
			finally:
					os.remove(lowerCamelCase_					)

			# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
			# the current folder and have the right name.
			if os.path.isfile('''tokenizer.json'''					):
					# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
					return
			try:
					with open('''tokenizer.json'''		, '''wb'''					) as f:
							http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''		, lowerCamelCase_					)
					SCREAMING_SNAKE_CASE       :     str				= AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''					)
					# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
					self.assertEqual(tokenizer.vocab_size		, 10_00					)
					# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

			finally:
					os.remove('''tokenizer.json'''					)







	def   __lowerCAmelCase (						self    :List[Any]					)							->			Union[str, Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[Any]				= AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''					)





@is_staging_test
class 				lowercase__(				unittest.TestCase   ):
	'''simple docstring'''

	UpperCamelCase				=  ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]





	@classmethod
	def   __lowerCAmelCase (						cls    :Optional[int]					)							->			List[str]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     str				= TOKEN
			HfFolder.save_token(lowerCamelCase_					)





	@classmethod
	def   __lowerCAmelCase (						cls    :int					)							->			Any:







			'''simple docstring'''




			try:
					delete_repo(token=cls._token		, repo_id='''test-tokenizer'''					)
			except HTTPError:
					pass

			try:
					delete_repo(token=cls._token		, repo_id='''valid_org/test-tokenizer-org'''					)
			except HTTPError:
					pass

			try:
					delete_repo(token=cls._token		, repo_id='''test-dynamic-tokenizer'''					)
			except HTTPError:
					pass





	def   __lowerCAmelCase (						self    :str					)							->			Tuple:







			'''simple docstring'''




			with tempfile.TemporaryDirectory() as tmp_dir:
					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= os.path.join(lowerCamelCase_		, '''vocab.txt'''					)
					with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''					) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
					SCREAMING_SNAKE_CASE       :     Tuple				= BertTokenizer(lowerCamelCase_					)

			tokenizer.push_to_hub('''test-tokenizer'''		, use_auth_token=self._token					)
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"					)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab					)

			# Reset repo
			delete_repo(token=self._token		, repo_id='''test-tokenizer'''					)

			# Push to hub via save_pretrained
			with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(lowerCamelCase_		, repo_id='''test-tokenizer'''		, push_to_hub=lowerCamelCase_		, use_auth_token=self._token					)

			SCREAMING_SNAKE_CASE       :     Optional[int]				= BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"					)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab					)





	def   __lowerCAmelCase (						self    :Optional[int]					)							->			str:







			'''simple docstring'''




			with tempfile.TemporaryDirectory() as tmp_dir:
					SCREAMING_SNAKE_CASE       :     int				= os.path.join(lowerCamelCase_		, '''vocab.txt'''					)
					with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''					) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= BertTokenizer(lowerCamelCase_					)

			tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''		, use_auth_token=self._token					)
			SCREAMING_SNAKE_CASE       :     Dict				= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''					)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab					)

			# Reset repo
			delete_repo(token=self._token		, repo_id='''valid_org/test-tokenizer-org'''					)

			# Push to hub via save_pretrained
			with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(
					    lowerCamelCase_		, repo_id='''valid_org/test-tokenizer-org'''		, push_to_hub=lowerCamelCase_		, use_auth_token=self._token					)

			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''					)
			self.assertDictEqual(new_tokenizer.vocab		, tokenizer.vocab					)







	@require_tokenizers
	def   __lowerCAmelCase (						self    :Optional[Any]					)							->			int:







			'''simple docstring'''




			CustomTokenizer.register_for_auto_class()
			with tempfile.TemporaryDirectory() as tmp_dir:
					SCREAMING_SNAKE_CASE       :     List[Any]				= os.path.join(lowerCamelCase_		, '''vocab.txt'''					)
					with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''					) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)
					SCREAMING_SNAKE_CASE       :     Optional[Any]				= CustomTokenizer(lowerCamelCase_					)

			# No fast custom tokenizer
			tokenizer.push_to_hub('''test-dynamic-tokenizer'''		, use_auth_token=self._token					)

			SCREAMING_SNAKE_CASE       :     Optional[Any]				= AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"		, trust_remote_code=lowerCamelCase_					)
			# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizer'''					)

			# Fast and slow custom tokenizer
			CustomTokenizerFast.register_for_auto_class()
			with tempfile.TemporaryDirectory() as tmp_dir:
					SCREAMING_SNAKE_CASE       :     Dict				= os.path.join(lowerCamelCase_		, '''vocab.txt'''					)
					with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''					) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]					)					)

					SCREAMING_SNAKE_CASE       :     Dict				= BertTokenizerFast.from_pretrained(lowerCamelCase_					)
					bert_tokenizer.save_pretrained(lowerCamelCase_					)
					SCREAMING_SNAKE_CASE       :     List[Any]				= CustomTokenizerFast.from_pretrained(lowerCamelCase_					)

			tokenizer.push_to_hub('''test-dynamic-tokenizer'''		, use_auth_token=self._token					)

			SCREAMING_SNAKE_CASE       :     List[Any]				= AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"		, trust_remote_code=lowerCamelCase_					)
			# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizerFast'''					)
			SCREAMING_SNAKE_CASE       :     List[Any]				= AutoTokenizer.from_pretrained(
			    f"{USER}/test-dynamic-tokenizer"		, use_fast=lowerCamelCase_		, trust_remote_code=lowerCamelCase_					)
			# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
			self.assertEqual(tokenizer.__class__.__name__		, '''CustomTokenizer'''					)





class 				lowercase__(				unittest.TestCase   ):
	'''simple docstring'''

	def   __lowerCAmelCase (						self    :Union[str, Any]					)							->			List[str]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= Trie()
			trie.add('''Hello 友達'''					)
			self.assertEqual(trie.data		, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}					)
			trie.add('''Hello'''					)
			trie.data
			self.assertEqual(trie.data		, {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}					)





	def   __lowerCAmelCase (						self    :str					)							->			Optional[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= Trie()
			self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''					)		, ['''[CLS] This is a extra_id_100''']					)
			trie.add('''[CLS]'''					)
			trie.add('''extra_id_1'''					)
			trie.add('''extra_id_100'''					)
			self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''					)		, ['''[CLS]''', ''' This is a ''', '''extra_id_100''']					)





	def   __lowerCAmelCase (						self    :Optional[Any]					)							->			int:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[Any]				= Trie()
			trie.add('''A'''					)
			self.assertEqual(trie.split('''ABC'''					)		, ['''A''', '''BC''']					)
			self.assertEqual(trie.split('''BCA'''					)		, ['''BC''', '''A''']					)





	def   __lowerCAmelCase (						self    :int					)							->			int:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[int]				= Trie()
			trie.add('''TOKEN]'''					)
			trie.add('''[SPECIAL_TOKEN]'''					)
			self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''					)		, ['''This is something ''', '''[SPECIAL_TOKEN]''']					)





	def   __lowerCAmelCase (						self    :List[Any]					)							->			Optional[int]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[Any]				= Trie()
			trie.add('''A'''					)
			trie.add('''P'''					)
			trie.add('''[SPECIAL_TOKEN]'''					)
			self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''					)		, ['''This is something ''', '''[SPECIAL_TOKEN]''']					)





	def   __lowerCAmelCase (						self    :int					)							->			Tuple:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[Any]				= Trie()
			trie.add('''AB'''					)
			trie.add('''B'''					)
			trie.add('''C'''					)
			self.assertEqual(trie.split('''ABC'''					)		, ['''AB''', '''C''']					)





	def   __lowerCAmelCase (						self    :Dict					)							->			List[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[Any]				= Trie()
			trie.add('''ABC'''					)
			trie.add('''B'''					)
			trie.add('''CD'''					)
			self.assertEqual(trie.split('''ABCD'''					)		, ['''ABC''', '''D''']					)







	def   __lowerCAmelCase (						self    :Optional[Any]					)							->			int:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     int				= Trie()
			SCREAMING_SNAKE_CASE       :     Optional[int]				= trie.cut_text('''ABC'''		, [0, 0, 2, 1, 2, 3]					)
			self.assertEqual(lowerCamelCase_		, ['''AB''', '''C''']					)







"""simple docstring"""






from __future__ import annotations

from numpy import array, cos, cross, floataa, radians, sin
from numpy.typing import NDArray





def __A						(   a_ :							float		,		a_ :							float		,		a_ :							bool = False     )->							list[float]:




		'''simple docstring'''






		if radian_mode:
				return [magnitude * cos(a_     ), magnitude * sin(a_     )]
		return [magnitude * cos(radians(a_     )     ), magnitude * sin(radians(a_     )     )]





def __A						(   a_ :							NDArray[floataa]		,		a_ :							NDArray[floataa]		,		a_ :							float = 10**-1     )->							bool:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     NDArray[floataa]				= cross(a_		,		a_     )
		SCREAMING_SNAKE_CASE       :     float				= sum(a_     )
		return abs(a_     ) < eps


if __name__ == "__main__":
					# Test to check if it works
					lowerCamelCase__ :       Optional[Any]						     =			array(
					    [
					        polar_force(7_1_8.4, 180 - 30),
					        polar_force(8_7_9.5_4, 45),
					        polar_force(100, -90),
					    ]
					)

					lowerCamelCase__ :       NDArray[floataa]						     =			array([[0, 0], [0, 0], [0, 0]])

					assert in_static_equilibrium(forces, location)

					# Problem 1 in image_data/2D_problems.jpg
					lowerCamelCase__ :       Union[str, Any]						     =			array(
					    [
					        polar_force(30 * 9.8_1, 15),
					        polar_force(215, 180 - 45),
					        polar_force(264, 90 - 30),
					    ]
					)

					lowerCamelCase__ :       Any						     =			array([[0, 0], [0, 0], [0, 0]])

					assert in_static_equilibrium(forces, location)

					# Problem in image_data/2D_problems_1.jpg
					lowerCamelCase__ :       Union[str, Any]						     =			array([[0, -2000], [0, -1200], [0, 15600], [0, -12400]])

					lowerCamelCase__ :       Optional[int]						     =			array([[0, 0], [6, 0], [10, 0], [12, 0]])

					assert in_static_equilibrium(forces, location)

					import doctest

					doctest.testmod()





