



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence([1, 2, 3]    )    )
           self.assertEqual(arr.type				,			pa.intaa()    )







     def 						_a      (self    ):






           '''simple docstring'''



           with self.assertRaises(__a    ):
                 lowerCamelCase								=		pa.array(TypedSequence([1, 2, 3]    )				,			type=pa.intaa()    )







     def 						_a      (self    ):






           '''simple docstring'''



           with self.assertRaises(__a    ):
                 lowerCamelCase								=		pa.array(TypedSequence([1, 2, 3]				,			try_type=Value("bool"    )				,			type=Value("int64"    )    )    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence([1, 2, 3]				,			type=Value("int32"    )    )    )
           self.assertEqual(arr.type				,			pa.intaa()    )







     def 						_a      (self    ):






           '''simple docstring'''



           with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
                 lowerCamelCase								=		pa.array(TypedSequence(["foo", "bar"]				,			type=Value("int64"    )    )    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence([1, 2, 3]				,			try_type=Value("int32"    )    )    )
           self.assertEqual(arr.type				,			pa.intaa()    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence(["foo", "bar"]				,			try_type=Value("int64"    )    )    )
           self.assertEqual(arr.type				,			pa.string()    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence([[[1, 2, 3]]]				,			type=ArrayaD((1, 3)				,			"int64"    )    )    )
           self.assertEqual(arr.type				,			ArrayaDExtensionType((1, 3)				,			"int64"    )    )







     def 						_a      (self    ):






           '''simple docstring'''



           with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
                 lowerCamelCase								=		pa.array(TypedSequence(["foo", "bar"]				,			type=ArrayaD((1, 3)				,			"int64"    )    )    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence([[[1, 2, 3]]]				,			try_type=ArrayaD((1, 3)				,			"int64"    )    )    )
           self.assertEqual(arr.type				,			ArrayaDExtensionType((1, 3)				,			"int64"    )    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pa.array(TypedSequence(["foo", "bar"]				,			try_type=ArrayaD((1, 3)				,			"int64"    )    )    )
           self.assertEqual(arr.type				,			pa.string()    )







     @require_pil
     def 						_a      (self    ):






           '''simple docstring'''



           import PIL.Image

           lowerCamelCase								=		PIL.Image.fromarray(np.arange(10				,			dtype=np.uinta    ).reshape(2				,			5    )    )
           with patch(
               "datasets.arrow_writer.cast_to_python_objects"				,			side_effect=__a    ) as mock_cast_to_python_objects:
                 lowerCamelCase								=		pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]				,			type=Image()    )    )
                 lowerCamelCase			,							lowerCamelCase								=		mock_cast_to_python_objects.call_args_list[-1]
                 self.assertIn("optimize_list_casting"				,			__a    )
                 self.assertFalse(kwargs["optimize_list_casting"]    )



def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferReader(UpperCAmelCase__ ) if isinstance(UpperCAmelCase__   ,   pa.Buffer ) else pa.memory_map(UpperCAmelCase__ )
      lowerCamelCase								=		pa.ipc.open_stream(UpperCAmelCase__ )
      lowerCamelCase								=		f.read_all()
      assert len(pa_table.to_batches() ) == expected_num_chunks
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
      del pa_table

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 1, 10] )
@pytest.mark.parametrize(
    "fields"   ,   [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      lowerCamelCase								=		pa.schema(UpperCAmelCase__ ) if fields else None
      with ArrowWriter(stream=UpperCAmelCase__   ,   schema=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__ ) as writer:
            writer.write({"col_1": "foo", "col_2": 1} )
            writer.write({"col_1": "bar", "col_2": 2} )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            lowerCamelCase								=		{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(UpperCAmelCase__   ,   metadata=writer._schema.metadata )
      _check_output(output.getvalue()   ,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )

def     __lowercase(	):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      lowerCamelCase								=		Features({"labels": ClassLabel(names=["neg", "pos"] )} )
      with ArrowWriter(stream=UpperCAmelCase__   ,   features=UpperCAmelCase__ ) as writer:
            writer.write({"labels": 0} )
            writer.write({"labels": 1} )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert writer._schema == features.arrow_schema
      assert writer._schema.metadata == features.arrow_schema.metadata
      lowerCamelCase								=		pa.BufferReader(output.getvalue() )
      lowerCamelCase								=		pa.ipc.open_stream(UpperCAmelCase__ )
      lowerCamelCase								=		f.read_all()
      lowerCamelCase								=		pa_table.schema
      assert pa_table.num_rows == 2
      assert schema == features.arrow_schema
      assert schema.metadata == features.arrow_schema.metadata
      assert features == Features.from_arrow_schema(UpperCAmelCase__ )

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 1, 10] )
def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      with ArrowWriter(
          stream=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__   ,   hash_salt="split_name"   ,   check_duplicates=UpperCAmelCase__   ,   ) as writer:
            with pytest.raises(UpperCAmelCase__ ):
                  writer.write({"col_1": "foo", "col_2": 1}   ,   key=[1, 2] )
                  lowerCamelCase			,							lowerCamelCase								=		writer.finalize()

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 2, 10] )
def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      with ArrowWriter(
          stream=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__   ,   hash_salt="split_name"   ,   check_duplicates=UpperCAmelCase__   ,   ) as writer:
            with pytest.raises(UpperCAmelCase__ ):
                  writer.write({"col_1": "foo", "col_2": 1}   ,   key=10 )
                  writer.write({"col_1": "bar", "col_2": 2}   ,   key=10 )
                  lowerCamelCase			,							lowerCamelCase								=		writer.finalize()

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 2, 10] )
def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      with ArrowWriter(
          stream=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__   ,   hash_salt="split_name"   ,   check_duplicates=UpperCAmelCase__   ,   ) as writer:
            writer.write({"col_1": "foo", "col_2": 1}   ,   key=1 )
            writer.write({"col_1": "bar", "col_2": 2}   ,   key=2 )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      _check_output(output.getvalue()   ,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 1, 10] )
@pytest.mark.parametrize(
    "fields"   ,   [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      lowerCamelCase								=		pa.schema(UpperCAmelCase__ ) if fields else None
      with ArrowWriter(stream=UpperCAmelCase__   ,   schema=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__ ) as writer:
            writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]} )
            writer.write_batch({"col_1": [], "col_2": []} )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            lowerCamelCase								=		{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(UpperCAmelCase__   ,   metadata=writer._schema.metadata )
      _check_output(output.getvalue()   ,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 1, 10] )
@pytest.mark.parametrize(
    "fields"   ,   [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      lowerCamelCase								=		pa.schema(UpperCAmelCase__ ) if fields else None
      with ArrowWriter(stream=UpperCAmelCase__   ,   schema=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__ ) as writer:
            writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]} ) )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            lowerCamelCase								=		{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(UpperCAmelCase__   ,   metadata=writer._schema.metadata )
      _check_output(output.getvalue()   ,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )

@pytest.mark.parametrize("writer_batch_size"   ,   [None, 1, 10] )
@pytest.mark.parametrize(
    "fields"   ,   [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      lowerCamelCase								=		pa.schema(UpperCAmelCase__ ) if fields else None
      with ArrowWriter(stream=UpperCAmelCase__   ,   schema=UpperCAmelCase__   ,   writer_batch_size=UpperCAmelCase__ ) as writer:
            writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]} ) )
            writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]} ) )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            lowerCamelCase								=		{"col_1": pa.string(), "col_2": pa.intaa()}
      assert writer._schema == pa.schema(UpperCAmelCase__   ,   metadata=writer._schema.metadata )
      _check_output(output.getvalue()   ,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )

def     __lowercase(	):


      """simple docstring"""



      with tempfile.TemporaryDirectory() as tmp_dir:
            lowerCamelCase								=		{"col_1": pa.string(), "col_2": pa.intaa()}
            lowerCamelCase								=		os.path.join(UpperCAmelCase__   ,   "test.arrow" )
            with ArrowWriter(path=UpperCAmelCase__   ,   schema=pa.schema(UpperCAmelCase__ ) ) as writer:
                  writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]} )
                  lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
            assert num_examples == 2
            assert num_bytes > 0
            assert writer._schema == pa.schema(UpperCAmelCase__   ,   metadata=writer._schema.metadata )
            _check_output(UpperCAmelCase__   ,   1 )

def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      if pa.types.is_list(UpperCAmelCase__ ):
            return get_base_dtype(arr_type.value_type )
      else:
            return arr_type

def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      if isinstance(lst[0]   ,   UpperCAmelCase__ ):
            change_first_primitive_element_in_list(lst[0]   ,   UpperCAmelCase__ )
      else:
            lowerCamelCase								=		value

@pytest.mark.parametrize("optimized_int_type, expected_dtype"   ,   [(None, pa.intaa()), (Value("int32" ), pa.intaa())] )
@pytest.mark.parametrize("sequence"   ,   [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.array(TypedSequence(UpperCAmelCase__   ,   optimized_int_type=UpperCAmelCase__ ) )
      assert get_base_dtype(arr.type ) == expected_dtype

@pytest.mark.parametrize(
    "col, expected_dtype"   ,   [
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]   ,   )
@pytest.mark.parametrize("sequence"   ,   [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		pa.array(OptimizedTypedSequence(UpperCAmelCase__   ,   col=UpperCAmelCase__ ) )
      assert get_base_dtype(arr.type ) == expected_dtype

      # not in range
      if col != "other":
            # avoids errors due to in-place modifications
            lowerCamelCase								=		copy.deepcopy(UpperCAmelCase__ )
            lowerCamelCase								=		np.iinfo(expected_dtype.to_pandas_dtype() ).max + 1
            change_first_primitive_element_in_list(UpperCAmelCase__   ,   UpperCAmelCase__ )
            lowerCamelCase								=		pa.array(OptimizedTypedSequence(UpperCAmelCase__   ,   col=UpperCAmelCase__ ) )
            assert get_base_dtype(arr.type ) == pa.intaa()

@pytest.mark.parametrize("raise_exception"   ,   [False, True] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		str(tmp_path / "dataset-train.arrow" )
      try:
            with ArrowWriter(path=UpperCAmelCase__ ) as writer:
                  if raise_exception:
                        raise pa.lib.ArrowInvalid()
                  else:
                        writer.stream.close()
      except pa.lib.ArrowInvalid:
            pass
      finally:
            assert writer.stream.closed

def     __lowercase(	UpperCAmelCase__ ):


      """simple docstring"""



      lowerCamelCase								=		"mock://dataset-train.arrow"
      with ArrowWriter(path=UpperCAmelCase__   ,   storage_options=mockfs.storage_options ) as writer:
            assert isinstance(writer._fs   ,   type(UpperCAmelCase__ ) )
            assert writer._fs.storage_options == mockfs.storage_options
            writer.write({"col_1": "foo", "col_2": 1} )
            writer.write({"col_1": "bar", "col_2": 2} )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert mockfs.exists(UpperCAmelCase__ )

def     __lowercase(	):


      """simple docstring"""



      lowerCamelCase								=		pa.BufferOutputStream()
      with ParquetWriter(stream=UpperCAmelCase__ ) as writer:
            writer.write({"col_1": "foo", "col_2": 1} )
            writer.write({"col_1": "bar", "col_2": 2} )
            lowerCamelCase			,							lowerCamelCase								=		writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      lowerCamelCase								=		pa.BufferReader(output.getvalue() )
      lowerCamelCase								=		pq.read_table(UpperCAmelCase__ )
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}

@require_pil
@pytest.mark.parametrize("embed_local_files"   ,   [False, True] )
def     __lowercase(	UpperCAmelCase__   ,   UpperCAmelCase__ ):


      """simple docstring"""



      import PIL.Image

      lowerCamelCase								=		str(tmp_path / "test_image_rgb.jpg" )
      PIL.Image.fromarray(np.zeros((5, 5)   ,   dtype=np.uinta ) ).save(UpperCAmelCase__   ,   format="png" )
      lowerCamelCase								=		pa.BufferOutputStream()
      with ParquetWriter(
          stream=UpperCAmelCase__   ,   features=Features({"image": Image()} )   ,   embed_local_files=UpperCAmelCase__ ) as writer:
            writer.write({"image": image_path} )
            writer.finalize()
      lowerCamelCase								=		pa.BufferReader(output.getvalue() )
      lowerCamelCase								=		pq.read_table(UpperCAmelCase__ )
      lowerCamelCase								=		pa_table.to_pydict()
      if embed_local_files:
            assert isinstance(out["image"][0]["path"]   ,   UpperCAmelCase__ )
            with open(UpperCAmelCase__   ,   "rb" ) as f:
                  assert out["image"][0]["bytes"] == f.read()
      else:
            assert out["image"][0]["path"] == image_path
            assert out["image"][0]["bytes"] is None






def     __lowercase(	):


      """simple docstring"""



      lowerCamelCase								=		pa.schema([pa.field("col_1"   ,   pa.string()   ,   nullable=UpperCAmelCase__ )] )
      lowerCamelCase								=		pa.BufferOutputStream()
      with ArrowWriter(stream=UpperCAmelCase__ ) as writer:
            writer._build_writer(inferred_schema=UpperCAmelCase__ )
            assert writer._schema == pa.schema([pa.field("col_1"   ,   pa.string() )] )




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		mock.Mock()
           lowerCamelCase								=		5_00
           lowerCamelCase								=		{}
           lowerCamelCase								=		HTTPError
           lowerCamelCase								=		{}

           # Download this model to make sure it's in the cache.
           lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("requests.Session.request"				,			return_value=__a    ) as mock_head:
                 lowerCamelCase								=		GPTaTokenizerFast.from_pretrained("gpt2"    )
                 # This check we did call the fake head request
                 mock_head.assert_called()







     def 						_a      (self    ):






           '''simple docstring'''



           try:
                 lowerCamelCase								=		tempfile.mktemp()
                 with open(__a				,			"wb"    ) as f:
                       http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"				,			__a    )

                 lowerCamelCase								=		AlbertTokenizer.from_pretrained(__a    )
           finally:
                 os.remove(__a    )

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile("tokenizer.json"    ):
                 # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                 return
           try:
                 with open("tokenizer.json"				,			"wb"    ) as f:
                       http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"				,			__a    )
                 lowerCamelCase								=		AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"    )
                 # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                 self.assertEqual(tokenizer.vocab_size				,			10_00    )
                 # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
                 os.remove("tokenizer.json"    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"    )





@is_staging_test
class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     _A       =			['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           lowerCamelCase								=		TOKEN
           HfFolder.save_token(__a    )







     @classmethod
     def 						_a      (cls    ):






           '''simple docstring'''



           try:
                 delete_repo(token=cls._token				,			repo_id="test-tokenizer"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="valid_org/test-tokenizer-org"    )
           except HTTPError:
                 pass

           try:
                 delete_repo(token=cls._token				,			repo_id="test-dynamic-tokenizer"    )
           except HTTPError:
                 pass







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("test-tokenizer"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="test-tokenizer"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__a				,			repo_id="test-tokenizer"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     def 						_a      (self    ):






           '''simple docstring'''



           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		BertTokenizer(__a    )

           tokenizer.push_to_hub("valid_org/test-tokenizer-org"				,			use_auth_token=self._token    )
           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )

           # Reset repo
           delete_repo(token=self._token				,			repo_id="valid_org/test-tokenizer-org"    )

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(
                     __a				,			repo_id="valid_org/test-tokenizer-org"				,			push_to_hub=__a				,			use_auth_token=self._token    )

           lowerCamelCase								=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"    )
           self.assertDictEqual(new_tokenizer.vocab				,			tokenizer.vocab    )







     @require_tokenizers
     def 						_a      (self    ):






           '''simple docstring'''



           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )
                 lowerCamelCase								=		CustomTokenizer(__a    )

           # No fast custom tokenizer
           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase								=		os.path.join(__a				,			"vocab.txt"    )
                 with open(__a				,			"w"				,			encoding="utf-8"    ) as vocab_writer:
                       vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]    )    )

                 lowerCamelCase								=		BertTokenizerFast.from_pretrained(__a    )
                 bert_tokenizer.save_pretrained(__a    )
                 lowerCamelCase								=		CustomTokenizerFast.from_pretrained(__a    )

           tokenizer.push_to_hub("test-dynamic-tokenizer"				,			use_auth_token=self._token    )

           lowerCamelCase								=		AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizerFast"    )
           lowerCamelCase								=		AutoTokenizer.from_pretrained(
               F"""{USER}/test-dynamic-tokenizer"""				,			use_fast=__a				,			trust_remote_code=__a    )
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__				,			"CustomTokenizer"    )





class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("Hello 友達"    )
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}    )
           trie.add("Hello"    )
           trie.data
           self.assertEqual(trie.data				,			{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS] This is a extra_id_100"]    )
           trie.add("[CLS]"    )
           trie.add("extra_id_1"    )
           trie.add("extra_id_100"    )
           self.assertEqual(trie.split("[CLS] This is a extra_id_100"    )				,			["[CLS]", " This is a ", "extra_id_100"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           self.assertEqual(trie.split("ABC"    )				,			["A", "BC"]    )
           self.assertEqual(trie.split("BCA"    )				,			["BC", "A"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("TOKEN]"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("A"    )
           trie.add("P"    )
           trie.add("[SPECIAL_TOKEN]"    )
           self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"    )				,			["This is something ", "[SPECIAL_TOKEN]"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("AB"    )
           trie.add("B"    )
           trie.add("C"    )
           self.assertEqual(trie.split("ABC"    )				,			["AB", "C"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           trie.add("ABC"    )
           trie.add("B"    )
           trie.add("CD"    )
           self.assertEqual(trie.split("ABCD"    )				,			["ABC", "D"]    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		Trie()
           lowerCamelCase								=		trie.cut_text("ABC"				,			[0, 0, 2, 1, 2, 3]    )
           self.assertEqual(__a				,			["AB", "C"]    )