


'''simple docstring'''

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     a_						:       List[Any]      			=				"""pt"""
elif is_tf_available():
     a_						:       str      			=				"""tf"""
else:
     a_						:       Dict      			=				"""jax"""





class   snake_case					(					lowercase    ,		unittest.TestCase    ):





  """simple docstring"""




  _lowerCamelCase						=						ByTaTokenizer
  _lowerCamelCase						=						False






  def 						snake_case	(					self  ):







   """simple docstring"""
   super().setUp()
   lowerCamelCase_  				=						ByTaTokenizer()
   tokenizer.save_pretrained(self.tmpdirname  )






  @cached_property
  def 						snake_case	(					self  ):







   """simple docstring"""
   return ByTaTokenizer.from_pretrained("google/byt5-small"  )






  def 						snake_case	(					self    ,						**UpperCamelCase  ):







   """simple docstring"""
   return self.tokenizer_class.from_pretrained(self.tmpdirname    ,						**UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase=False    ,						UpperCamelCase=20    ,						UpperCamelCase=5  ):







   """simple docstring"""
   # XXX The default common tokenizer tests assume that every ID is decodable on its own.
   # This assumption is invalid for ByT5 because single bytes might not be
   # valid utf-8 (byte 128 for instance).
   # Here we're overriding the smallest possible method to provide
   # a clean sequence without making the same assumption.

   lowerCamelCase_  				=						[]
   for i in range(len(UpperCamelCase  )  ):
    try:
     lowerCamelCase_  				=						tokenizer.decode([i]    ,						clean_up_tokenization_spaces=UpperCamelCase  )
    except UnicodeDecodeError:
     pass
    toks.append((i, tok)  )

   lowerCamelCase_  				=						list(filter(lambda UpperCamelCase  : re.match(r"^[ a-zA-Z]+$"    ,						t[1]  )    ,						UpperCamelCase  )  )
   lowerCamelCase_  				=						list(filter(lambda UpperCamelCase  : [t[0]] == tokenizer.encode(t[1]    ,						add_special_tokens=UpperCamelCase  )    ,						UpperCamelCase  )  )
   if max_length is not None and len(UpperCamelCase  ) > max_length:
    lowerCamelCase_  				=						toks[:max_length]
   if min_length is not None and len(UpperCamelCase  ) < min_length and len(UpperCamelCase  ) > 0:
    while len(UpperCamelCase  ) < min_length:
     lowerCamelCase_  				=						toks + toks
        # toks_str = [t[1] for t in toks]
   lowerCamelCase_  				=						[t[0] for t in toks]

   # Ensure consistency
   lowerCamelCase_  				=						tokenizer.decode(UpperCamelCase    ,						clean_up_tokenization_spaces=UpperCamelCase  )
   if " " not in output_txt and len(UpperCamelCase  ) > 1:
    lowerCamelCase_  				=						(
        tokenizer.decode([toks_ids[0]]    ,						clean_up_tokenization_spaces=UpperCamelCase  )
        + " "
        + tokenizer.decode(toks_ids[1:]    ,						clean_up_tokenization_spaces=UpperCamelCase  )
    )
   if with_prefix_space:
    lowerCamelCase_  				=						" " + output_txt
   lowerCamelCase_  				=						tokenizer.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  )
   return output_txt, output_ids






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]  )
   lowerCamelCase_  				=						tokenizer(["hi", "I went to the gym", ""]  )
   self.assertListEqual(batch_with_eos_added["input_ids"]    ,						batch_without_eos_added["input_ids"]  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						"Unicode €."
   lowerCamelCase_  				=						tokenizer(UpperCamelCase  )
   lowerCamelCase_  				=						[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
   self.assertEqual(encoded["input_ids"]    ,						UpperCamelCase  )

   # decoding
   lowerCamelCase_  				=						tokenizer.decode(UpperCamelCase  )
   self.assertEqual(UpperCamelCase    ,						"Unicode €.</s>"  )

   lowerCamelCase_  				=						tokenizer("e è é ê ë"  )
   lowerCamelCase_  				=						[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
   self.assertEqual(encoded["input_ids"]    ,						UpperCamelCase  )
   # decoding
   lowerCamelCase_  				=						tokenizer.decode(UpperCamelCase  )
   self.assertEqual(UpperCamelCase    ,						"e è é ê ë</s>"  )

   # encode/decode, but with `encode` instead of `__call__`
   self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"  )  )    ,						"e è é ê ë</s>"  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						["A long paragraph for summarization.", "Another paragraph for summarization."]
   # fmt: off
   lowerCamelCase_  				=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
   # fmt: on
   lowerCamelCase_  				=						tokenizer(UpperCamelCase    ,						padding=UpperCamelCase    ,						return_tensors=UpperCamelCase  )
   self.assertIsInstance(UpperCamelCase    ,						UpperCamelCase  )

   if FRAMEWORK != "jax":
    lowerCamelCase_  				=						list(batch.input_ids.numpy()[0]  )
   else:
    lowerCamelCase_  				=						list(batch.input_ids.tolist()[0]  )

   self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )

   self.assertEqual((2, 37)    ,						batch.input_ids.shape  )
   self.assertEqual((2, 37)    ,						batch.attention_mask.shape  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						["A long paragraph for summarization.", "Another paragraph for summarization."]
   lowerCamelCase_  				=						tokenizer(UpperCamelCase    ,						padding=UpperCamelCase    ,						return_tensors=UpperCamelCase  )
   # check if input_ids are returned and no decoder_input_ids
   self.assertIn("input_ids"    ,						UpperCamelCase  )
   self.assertIn("attention_mask"    ,						UpperCamelCase  )
   self.assertNotIn("decoder_input_ids"    ,						UpperCamelCase  )
   self.assertNotIn("decoder_attention_mask"    ,						UpperCamelCase  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						[
       "Summary of the text.",
       "Another summary.",
   ]
   lowerCamelCase_  				=						tokenizer(
       text_target=UpperCamelCase    ,						max_length=32    ,						padding="max_length"    ,						truncation=UpperCamelCase    ,						return_tensors=UpperCamelCase  )
   self.assertEqual(32    ,						targets["input_ids"].shape[1]  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.ta_base_tokenizer
   lowerCamelCase_  				=						["A long paragraph for summarization. </s>"]
   lowerCamelCase_  				=						["Summary of the text. </s>"]
   # fmt: off
   lowerCamelCase_  				=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
   lowerCamelCase_  				=						[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
   # fmt: on

   lowerCamelCase_  				=						tokenizer(UpperCamelCase    ,						text_target=UpperCamelCase  )

   self.assertEqual(UpperCamelCase    ,						batch["input_ids"][0]  )
   self.assertEqual(UpperCamelCase    ,						batch["labels"][0]  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   # safety check on max_len default value so we are sure the test works
   lowerCamelCase_  				=						self.get_tokenizers()
   for tokenizer in tokenizers:
    with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
     self.assertNotEqual(tokenizer.model_max_length    ,						42  )

        # Now let's start the test
   lowerCamelCase_  				=						self.get_tokenizers()
   for tokenizer in tokenizers:
    with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
     # Isolate this from the other tests because we save additional tokens/etc
     lowerCamelCase_  				=						tempfile.mkdtemp()

     lowerCamelCase_  				=						" He is very happy, UNwant\u00E9d,running"
     lowerCamelCase_  				=						tokenizer.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  )
     tokenizer.save_pretrained(UpperCamelCase  )

     lowerCamelCase_  				=						tokenizer.__class__.from_pretrained(UpperCamelCase  )
     lowerCamelCase_  				=						after_tokenizer.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  )
     self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )

     shutil.rmtree(UpperCamelCase  )

   lowerCamelCase_  				=						self.get_tokenizers(model_max_length=42  )
   for tokenizer in tokenizers:
    with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
     # Isolate this from the other tests because we save additional tokens/etc
     lowerCamelCase_  				=						tempfile.mkdtemp()

     lowerCamelCase_  				=						" He is very happy, UNwant\u00E9d,running"
     tokenizer.add_tokens(["bim", "bambam"]  )
     lowerCamelCase_  				=						tokenizer.additional_special_tokens
     additional_special_tokens.append("new_additional_special_token"  )
     tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}  )
     lowerCamelCase_  				=						tokenizer.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  )
     tokenizer.save_pretrained(UpperCamelCase  )

     lowerCamelCase_  				=						tokenizer.__class__.from_pretrained(UpperCamelCase  )
     lowerCamelCase_  				=						after_tokenizer.encode(UpperCamelCase    ,						add_special_tokens=UpperCamelCase  )
     self.assertListEqual(UpperCamelCase    ,						UpperCamelCase  )
     self.assertIn("new_additional_special_token"    ,						after_tokenizer.additional_special_tokens  )
     self.assertEqual(after_tokenizer.model_max_length    ,						42  )

     lowerCamelCase_  				=						tokenizer.__class__.from_pretrained(UpperCamelCase    ,						model_max_length=43  )
     self.assertEqual(tokenizer.model_max_length    ,						43  )

     shutil.rmtree(UpperCamelCase  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						[]
   if self.test_slow_tokenizer:
    tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

   if self.test_rust_tokenizer:
    tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

   for tokenizer_class, tokenizer_utils in tokenizer_list:
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer_utils.save_pretrained(UpperCamelCase  )

     with open(os.path.join(UpperCamelCase    ,						"special_tokens_map.json"  )    ,						encoding="utf-8"  ) as json_file:
      lowerCamelCase_  				=						json.load(UpperCamelCase  )

     with open(os.path.join(UpperCamelCase    ,						"tokenizer_config.json"  )    ,						encoding="utf-8"  ) as json_file:
      lowerCamelCase_  				=						json.load(UpperCamelCase  )

     lowerCamelCase_  				=						[f'''<extra_id_{i}>''' for i in range(125  )]

     lowerCamelCase_  				=						added_tokens_extra_ids + [
         "an_additional_special_token"
     ]
     lowerCamelCase_  				=						added_tokens_extra_ids + [
         "an_additional_special_token"
     ]

     with open(os.path.join(UpperCamelCase    ,						"special_tokens_map.json"  )    ,						"w"    ,						encoding="utf-8"  ) as outfile:
      json.dump(UpperCamelCase    ,						UpperCamelCase  )
     with open(os.path.join(UpperCamelCase    ,						"tokenizer_config.json"  )    ,						"w"    ,						encoding="utf-8"  ) as outfile:
      json.dump(UpperCamelCase    ,						UpperCamelCase  )

     # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
     # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
     # "special_tokens_map.json" files
     lowerCamelCase_  				=						tokenizer_class.from_pretrained(
         UpperCamelCase    ,						)
     self.assertIn(
         "an_additional_special_token"    ,						tokenizer_without_change_in_init.additional_special_tokens  )
     # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
     self.assertEqual(
         ["an_additional_special_token"]    ,						tokenizer_without_change_in_init.convert_ids_to_tokens(
             tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]  )  )    ,						)

     # Now we test that we can change the value of additional_special_tokens in the from_pretrained
     lowerCamelCase_  				=						added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"    ,						lstrip=UpperCamelCase  )]
     lowerCamelCase_  				=						tokenizer_class.from_pretrained(
         UpperCamelCase    ,						additional_special_tokens=UpperCamelCase    ,						)

     self.assertIn("a_new_additional_special_token"    ,						tokenizer.additional_special_tokens  )
     self.assertEqual(
         ["a_new_additional_special_token"]    ,						tokenizer.convert_ids_to_tokens(
             tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]  )  )    ,						)






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						[]
   if self.test_slow_tokenizer:
    tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

   if self.test_rust_tokenizer:
    tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

   for tokenizer_class, tokenizer_utils in tokenizer_list:
    with tempfile.TemporaryDirectory() as tmp_dir:
     tokenizer_utils.save_pretrained(UpperCamelCase  )

     lowerCamelCase_  				=						tokenizer_class.from_pretrained(UpperCamelCase  )

     self.assertTrue(tokenizer.decode([255]  ) == ""  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self  ):







   """simple docstring"""
   # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
   # and special added tokens as tokens
   lowerCamelCase_  				=						self.get_tokenizers(fast=UpperCamelCase    ,						do_lower_case=UpperCamelCase  )
   for tokenizer in tokenizers:
    with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
     lowerCamelCase_  				=						["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
     lowerCamelCase_  				=						tokenizer.convert_tokens_to_string(UpperCamelCase  )

     self.assertIsInstance(UpperCamelCase    ,						UpperCamelCase  )







  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						self.get_tokenizers()
   for tokenizer in tokenizers:
    with self.subTest(f'''{tokenizer.__class__.__name__}'''  ):
     lowerCamelCase_  				=						[
         "bos_token",
         "eos_token",
         "unk_token",
         "sep_token",
         "pad_token",
         "cls_token",
         "mask_token",
     ]

     lowerCamelCase_  				=						0
     lowerCamelCase_  				=						tokenizer.convert_ids_to_tokens(
         UpperCamelCase    ,						skip_special_tokens=UpperCamelCase  )

     for attr in attributes_list:
      setattr(UpperCamelCase    ,						attr + "_id"    ,						UpperCamelCase  )
      self.assertEqual(getattr(UpperCamelCase    ,						UpperCamelCase  )    ,						UpperCamelCase  )
      self.assertEqual(getattr(UpperCamelCase    ,						attr + "_id"  )    ,						UpperCamelCase  )

      setattr(UpperCamelCase    ,						attr + "_id"    ,						UpperCamelCase  )
      self.assertEqual(getattr(UpperCamelCase    ,						UpperCamelCase  )    ,						UpperCamelCase  )
      self.assertEqual(getattr(UpperCamelCase    ,						attr + "_id"  )    ,						UpperCamelCase  )

     setattr(UpperCamelCase    ,						"additional_special_tokens_ids"    ,						[]  )
     self.assertListEqual(getattr(UpperCamelCase    ,						"additional_special_tokens"  )    ,						[]  )
     self.assertListEqual(getattr(UpperCamelCase    ,						"additional_special_tokens_ids"  )    ,						[]  )

     setattr(UpperCamelCase    ,						"additional_special_tokens_ids"    ,						[token_id_to_test_setters]  )
     self.assertListEqual(getattr(UpperCamelCase    ,						"additional_special_tokens"  )    ,						[token_to_test_setters]  )
     self.assertListEqual(getattr(UpperCamelCase    ,						"additional_special_tokens_ids"  )    ,						[token_id_to_test_setters]  )







'''simple docstring'''

# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


a_						:       Dict      			=				get_logger(__name__)





class   snake_case					:





  """simple docstring"""




  _lowerCamelCase						=						"dummy_data"
  _lowerCamelCase						=						"datasets"
  _lowerCamelCase						=						False






  def __init__(					self    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase    ,						UpperCamelCase = None    ,						UpperCamelCase = False    ,						UpperCamelCase = True    ,						UpperCamelCase = None    ,						):







   """simple docstring"""
   lowerCamelCase_  				=						0
   lowerCamelCase_  				=						dataset_name
   lowerCamelCase_  				=						cache_dir
   lowerCamelCase_  				=						use_local_dummy_data
   lowerCamelCase_  				=						config
   # download_callbacks take a single url as input
   lowerCamelCase_  				=						download_callbacks or []
   # if False, it doesn't load existing files and it returns the paths of the dummy files relative
   # to the dummy_data zip file root
   lowerCamelCase_  				=						load_existing_dummy_data

   # TODO(PVP, QL) might need to make this more general
   lowerCamelCase_  				=						str(UpperCamelCase  )
   # to be downloaded
   lowerCamelCase_  				=						None
   lowerCamelCase_  				=						None






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   if self._dummy_file is None:
    lowerCamelCase_  				=						self.download_dummy_data()
   return self._dummy_file






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   if self.config is not None:
    # structure is dummy / config_name / version_name
    return os.path.join("dummy"    ,						self.config.name    ,						self.version_name  )
   # structure is dummy / version_name
   return os.path.join("dummy"    ,						self.version_name  )






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   return os.path.join(self.dummy_data_folder    ,						"dummy_data.zip"  )






  def 						snake_case	(					self  ):







   """simple docstring"""
   lowerCamelCase_  				=						(
       self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
   )

   lowerCamelCase_  				=						cached_path(
       UpperCamelCase    ,						cache_dir=self.cache_dir    ,						extract_compressed_file=UpperCamelCase    ,						force_extract=UpperCamelCase  )

   return os.path.join(UpperCamelCase    ,						self.dummy_file_name  )






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   return os.path.join(self.datasets_scripts_dir    ,						self.dataset_name    ,						self.dummy_zip_file  )






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   if self._bucket_url is None:
    lowerCamelCase_  				=						hf_github_url(self.dataset_name    ,						self.dummy_zip_file.replace(os.sep    ,						"/"  )  )
   return self._bucket_url






  @property
  def 						snake_case	(					self  ):







   """simple docstring"""
   # return full path if its a dir
   if os.path.isdir(self.dummy_file  ):
    return self.dummy_file
   # else cut off path to file -> example `xsum`.
   return "/".join(self.dummy_file.replace(os.sep    ,						"/"  ).split("/"  )[:-1]  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						*UpperCamelCase  ):







   """simple docstring"""
   if self.load_existing_dummy_data:
    # dummy data is downloaded and tested
    lowerCamelCase_  				=						self.dummy_file
   else:
    # dummy data cannot be downloaded and only the path to dummy file is returned
    lowerCamelCase_  				=						self.dummy_file_name

   # special case when data_url is a dict
   if isinstance(UpperCamelCase    ,						UpperCamelCase  ):
    return self.create_dummy_data_dict(UpperCamelCase    ,						UpperCamelCase  )
   elif isinstance(UpperCamelCase    ,						(list, tuple)  ):
    return self.create_dummy_data_list(UpperCamelCase    ,						UpperCamelCase  )
   else:
    return self.create_dummy_data_single(UpperCamelCase    ,						UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						*UpperCamelCase  ):







   """simple docstring"""
   return self.download_and_extract(UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   return self.download_and_extract(UpperCamelCase  )






  def 						snake_case	(					self    ,						UpperCamelCase    ,						*UpperCamelCase    ,						**UpperCamelCase  ):







   """simple docstring"""
   return path






  def 						snake_case	(					self  ):







   """simple docstring"""
   return {}






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						{}
   for key, single_urls in data_url.items():
    for download_callback in self.download_callbacks:
     if isinstance(UpperCamelCase    ,						UpperCamelCase  ):
      for single_url in single_urls:
       download_callback(UpperCamelCase  )
     else:
      lowerCamelCase_  				=						single_urls
      download_callback(UpperCamelCase  )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
    if isinstance(UpperCamelCase    ,						UpperCamelCase  ):
     lowerCamelCase_  				=						[os.path.join(UpperCamelCase    ,						urllib.parse.quote_plus(Path(UpperCamelCase  ).name  )  ) for x in single_urls]
    else:
     lowerCamelCase_  				=						single_urls
     lowerCamelCase_  				=						os.path.join(UpperCamelCase    ,						urllib.parse.quote_plus(Path(UpperCamelCase  ).name  )  )
    lowerCamelCase_  				=						value

   # make sure that values are unique
   if all(isinstance(UpperCamelCase    ,						UpperCamelCase  ) for i in dummy_data_dict.values()  ) and len(set(dummy_data_dict.values()  )  ) < len(
       dummy_data_dict.values()  ):
    # append key to value to make its name unique
    lowerCamelCase_  				=						{key: value + key for key, value in dummy_data_dict.items()}

   return dummy_data_dict






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   lowerCamelCase_  				=						[]
   # trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
   lowerCamelCase_  				=						all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}"    ,						UpperCamelCase  )  ) for url in data_url  )
   lowerCamelCase_  				=						all(
       url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"  ) for url in data_url  )
   if data_url and (is_tf_records or is_pubmed_records):
    lowerCamelCase_  				=						[data_url[0]] * len(UpperCamelCase  )
   for single_url in data_url:
    for download_callback in self.download_callbacks:
     download_callback(UpperCamelCase  )
    # we force the name of each key to be the last file / folder name of the url path
    # if the url has arguments, we need to encode them with urllib.parse.quote_plus
    lowerCamelCase_  				=						os.path.join(UpperCamelCase    ,						urllib.parse.quote_plus(single_url.split("/"  )[-1]  )  )
    dummy_data_list.append(UpperCamelCase  )
   return dummy_data_list






  def 						snake_case	(					self    ,						UpperCamelCase    ,						UpperCamelCase  ):







   """simple docstring"""
   for download_callback in self.download_callbacks:
    download_callback(UpperCamelCase  )
   # we force the name of each key to be the last file / folder name of the url path
   # if the url has arguments, we need to encode them with urllib.parse.quote_plus
   lowerCamelCase_  				=						os.path.join(UpperCamelCase    ,						urllib.parse.quote_plus(data_url.split("/"  )[-1]  )  )
   if os.path.exists(UpperCamelCase  ) or not self.load_existing_dummy_data:
    return value
   else:
    # Backward compatibility, maybe deprecate at one point.
    # For many datasets with single url calls to dl_manager.download_and_extract,
    # the dummy_data.zip file is actually the zipped downloaded file
    # while now we expected the dummy_data.zip file to be a directory containing
    # the downloaded file.
    return path_to_dummy_data






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self  ):







   """simple docstring"""
   pass






  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   def _iter_archive_members(UpperCamelCase  ):
    # this preserves the order of the members inside the ZIP archive
    lowerCamelCase_  				=						Path(self.dummy_file  ).parent
    lowerCamelCase_  				=						path.relative_to(UpperCamelCase  )
    with ZipFile(self.local_path_to_dummy_data  ) as zip_file:
     lowerCamelCase_  				=						zip_file.namelist()
    for member in members:
     if member.startswith(relative_path.as_posix()  ):
      yield dummy_parent_path.joinpath(UpperCamelCase  )

   lowerCamelCase_  				=						Path(UpperCamelCase  )
   lowerCamelCase_  				=						_iter_archive_members(UpperCamelCase  ) if self.use_local_dummy_data else path.rglob("*"  )
   for file_path in file_paths:
    if file_path.is_file() and not file_path.name.startswith((".", "__")  ):
     yield file_path.relative_to(UpperCamelCase  ).as_posix(), file_path.open("rb"  )







  def 						snake_case	(					self    ,						UpperCamelCase  ):







   """simple docstring"""
   if not isinstance(UpperCamelCase    ,						UpperCamelCase  ):
    lowerCamelCase_  				=						[paths]
   for path in paths:
    if os.path.isfile(UpperCamelCase  ):
     if os.path.basename(UpperCamelCase  ).startswith((".", "__")  ):
      return
     yield path
    else:
     for dirpath, dirnames, filenames in os.walk(UpperCamelCase  ):
      if os.path.basename(UpperCamelCase  ).startswith((".", "__")  ):
       continue
      dirnames.sort()
      for filename in sorted(UpperCamelCase  ):
       if filename.startswith((".", "__")  ):
        continue
       yield os.path.join(UpperCamelCase    ,						UpperCamelCase  )



