






'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers.models.xglm.modeling_tf_xglm import (
						    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFXGLMForCausalLM,
						    TFXGLMModel,
						)






@require_tf
class 							__magic_name__     :







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      List[str]        =  XGLMConfig
	SCREAMING_SNAKE_CASE_					:      List[Any]        =  {}
	SCREAMING_SNAKE_CASE_					:      Optional[Any]        =  'gelu'
	def __init__(							self  ,      _lowercase  ,      _lowercase=14  ,      _lowercase=7  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=99  ,      _lowercase=32  ,      _lowercase=2  ,      _lowercase=4  ,      _lowercase=37  ,      _lowercase="gelu"  ,      _lowercase=0.1  ,      _lowercase=0.1  ,      _lowercase=512  ,      _lowercase=0.02  ,      )						->  str:
							lowercase_      :   Any        =			parent
							lowercase_      :   Tuple        =			batch_size
							lowercase_      :   Dict        =			seq_length
							lowercase_      :   List[str]        =			is_training
							lowercase_      :   int        =			use_input_mask
							lowercase_      :   Any        =			use_labels
							lowercase_      :   str        =			vocab_size
							lowercase_      :   Dict        =			d_model
							lowercase_      :   str        =			num_hidden_layers
							lowercase_      :   Optional[int]        =			num_attention_heads
							lowercase_      :   List[Any]        =			ffn_dim
							lowercase_      :   Optional[int]        =			activation_function
							lowercase_      :   int        =			activation_dropout
							lowercase_      :   Optional[Any]        =			attention_dropout
							lowercase_      :   List[str]        =			max_position_embeddings
							lowercase_      :   Optional[int]        =			initializer_range
							lowercase_      :   str        =			None
							lowercase_      :   Any        =			0
							lowercase_      :   Union[str, Any]        =			2
							lowercase_      :   Optional[Any]        =			1
	def        snake_case__		(							self       )						->  int:
							return XGLMConfig.from_pretrained('facebook/xglm-564M'       )
	def        snake_case__		(							self       )						->  Any:
							lowercase_      :   List[str]        =			tf.clip_by_value(
							    ids_tensor([self.batch_size, self.seq_length]  ,      self.vocab_size       )  ,      clip_value_min=0  ,      clip_value_max=3       )

							lowercase_      :   Optional[Any]        =			None
							if self.use_input_mask:
													lowercase_      :   int        =			random_attention_mask([self.batch_size, self.seq_length]       )

							lowercase_      :   str        =			self.get_config()

							lowercase_      :   List[Any]        =			floats_tensor([self.num_hidden_layers, self.num_attention_heads]  ,      2       )

							return (
							    config,
							    input_ids,
							    input_mask,
							    head_mask,
							)
	def        snake_case__		(							self       )						->  Any:
							return XGLMConfig(
							    vocab_size=self.vocab_size  ,      d_model=self.hidden_size  ,      num_layers=self.num_hidden_layers  ,      attention_heads=self.num_attention_heads  ,      ffn_dim=self.ffn_dim  ,      activation_function=self.activation_function  ,      activation_dropout=self.activation_dropout  ,      attention_dropout=self.attention_dropout  ,      max_position_embeddings=self.max_position_embeddings  ,      initializer_range=self.initializer_range  ,      use_cache=_lowercase  ,      bos_token_id=self.bos_token_id  ,      eos_token_id=self.eos_token_id  ,      pad_token_id=self.pad_token_id  ,      return_dict=_lowercase  ,      )







	def        snake_case__		(							self       )						->  Dict:
							lowercase_      :   str        =			self.prepare_config_and_inputs()

							(
							    lowercase_      
							)      :   Dict        =			config_and_inputs

							lowercase_      :   Optional[Any]        =			{
							    'input_ids': input_ids,
							    'head_mask': head_mask,
							}

							return config, inputs_dict






@require_tf
class 							__magic_name__     (	UpperCAmelCase_,						UpperCAmelCase_,						unittest.TestCase				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      int        =  (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
	SCREAMING_SNAKE_CASE_					:      Dict        =  (TFXGLMForCausalLM,) if is_tf_available() else ()
	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  (
	    {'feature-extraction': TFXGLMModel, 'text-generation': TFXGLMForCausalLM} if is_tf_available() else {}
	)
	SCREAMING_SNAKE_CASE_					:      str        =  False
	SCREAMING_SNAKE_CASE_					:      List[Any]        =  False
	SCREAMING_SNAKE_CASE_					:      int        =  False
	def        snake_case__		(							self       )						->  List[str]:
							lowercase_      :   Optional[Any]        =			TFXGLMModelTester(self       )
							lowercase_      :   List[str]        =			ConfigTester(self  ,      config_class=_lowercase  ,      n_embd=37       )
	def        snake_case__		(							self       )						->  List[Any]:
							self.config_tester.run_common_tests()
	@slow
	def        snake_case__		(							self       )						->  int:
							for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowercase_      :   Any        =			TFXGLMModel.from_pretrained(_lowercase       )
													self.assertIsNotNone(_lowercase       )







	@unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'       )
	def        snake_case__		(							self       )						->  List[Any]:
							super().test_resize_token_embeddings()






@require_tf
class 							__magic_name__     (	unittest.TestCase				):







	"""simple docstring"""



	@slow
	def        snake_case__		(							self  ,      _lowercase=True       )						->  List[Any]:
							lowercase_      :   List[str]        =			TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'       )
							lowercase_      :   Tuple        =			tf.convert_to_tensor([[2, 268, 9865]]  ,      dtype=tf.intaa       )  # The dog
							# </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
							# fmt: off
							lowercase_      :   Any        =			[2, 268, 9865, 67, 11, 1988, 5_7252, 9865, 5, 984, 67, 1988, 21_3838, 1658, 53, 7_0446, 33, 6657, 278, 1581]
							# fmt: on
							lowercase_      :   Tuple        =			model.generate(_lowercase  ,      do_sample=_lowercase  ,      num_beams=1       )
							if verify_outputs:
													self.assertListEqual(output_ids[0].numpy().tolist()  ,      _lowercase       )
	@slow
	def        snake_case__		(							self       )						->  List[str]:
							lowercase_      :   int        =			XGLMTokenizer.from_pretrained('facebook/xglm-564M'       )
							lowercase_      :   Dict        =			TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'       )

							tf.random.set_seed(0       )
							lowercase_      :   Any        =			tokenizer('Today is a nice day and'  ,      return_tensors='tf'       )
							lowercase_      :   Any        =			tokenized.input_ids
							# forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
							with tf.device(':/CPU:0'       ):
													lowercase_      :   List[str]        =			model.generate(_lowercase  ,      do_sample=_lowercase  ,      seed=[7, 0]       )
							lowercase_      :   Optional[Any]        =			tokenizer.decode(output_ids[0]  ,      skip_special_tokens=_lowercase       )

							lowercase_      :   Optional[Any]        =			(
							    'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
							)
							self.assertEqual(_lowercase  ,      _lowercase       )







	@slow
	def        snake_case__		(							self       )						->  Union[str, Any]:
							lowercase_      :   Optional[Any]        =			TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'       )
							lowercase_      :   Optional[Any]        =			XGLMTokenizer.from_pretrained('facebook/xglm-564M'       )

							lowercase_      :   str        =			'left'

							# use different length sentences to test batching
							lowercase_      :   str        =			[
							    'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
							    'left-padding, such as in batched generation. The output for the sequence below should be the same '
							    'regardless of whether left padding is applied or not. When',
							    'Hello, my dog is a little',
							]

							lowercase_      :   List[Any]        =			tokenizer(_lowercase  ,      return_tensors='tf'  ,      padding=_lowercase       )
							lowercase_      :   Tuple        =			inputs['input_ids']

							lowercase_      :   Optional[int]        =			model.generate(input_ids=_lowercase  ,      attention_mask=inputs['attention_mask']  ,      max_new_tokens=12       )

							lowercase_      :   Optional[Any]        =			tokenizer(sentences[0]  ,      return_tensors='tf'       ).input_ids
							lowercase_      :   List[Any]        =			model.generate(input_ids=_lowercase  ,      max_new_tokens=12       )

							lowercase_      :   List[Any]        =			tokenizer(sentences[1]  ,      return_tensors='tf'       ).input_ids
							lowercase_      :   Tuple        =			model.generate(input_ids=_lowercase  ,      max_new_tokens=12       )

							lowercase_      :   Union[str, Any]        =			tokenizer.batch_decode(_lowercase  ,      skip_special_tokens=_lowercase       )
							lowercase_      :   Optional[Any]        =			tokenizer.decode(output_non_padded[0]  ,      skip_special_tokens=_lowercase       )
							lowercase_      :   Tuple        =			tokenizer.decode(output_padded[0]  ,      skip_special_tokens=_lowercase       )

							lowercase_      :   Dict        =			[
							    'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
							    'left-padding, such as in batched generation. The output for the sequence below should be the same '
							    'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
							    'a single',
							    'Hello, my dog is a little bit of a shy one, but he is very friendly',
							]
							self.assertListEqual(_lowercase  ,      _lowercase       )
							self.assertListEqual(_lowercase  ,      [non_padded_sentence, padded_sentence]       )








'''simple docstring'''

from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url






class 							__magic_name__     (	UpperCAmelCase_				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  ''
	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  'hf-legacy'  # "hf://"" is reserved for hffs
	def __init__(							self  ,      _lowercase = None  ,      _lowercase = None  ,      **_lowercase  ,      )						->  Optional[Any]:
							super().__init__(self  ,      **_lowercase       )
							lowercase_      :   int        =			repo_info
							lowercase_      :   List[Any]        =			token
							lowercase_      :   Union[str, Any]        =			None
	def        lowerCamelCase__		(							self       )						->  Optional[Any]:
							if self.dir_cache is None:
													lowercase_      :   Optional[Any]        =			{}
													for hf_file in self.repo_info.siblings:
																			# TODO(QL): add sizes
																			lowercase_      :   str        =			{
																			    'name': hf_file.rfilename,
																			    'size': None,
																			    'type': 'file',
																			}
																			self.dir_cache.update(
																			    {
																			        str(_lowercase       ): {'name': str(_lowercase       ), 'size': None, 'type': 'directory'}
																			        for d in list(PurePosixPath(hf_file.rfilename       ).parents       )[:-1]
																			    }       )
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase = "rb"  ,      **_lowercase  ,      )						->  Dict:
							if not isinstance(self.repo_info  ,      _lowercase       ):
													raise NotImplementedError(f"Open is only implemented for dataset repositories, but got {self.repo_info}"       )
							lowercase_      :   Optional[int]        =			hf_hub_url(self.repo_info.id  ,      _lowercase  ,      revision=self.repo_info.sha       )
							return fsspec.open(
							    _lowercase  ,      mode=_lowercase  ,      headers=get_authentication_headers_for_url(_lowercase  ,      use_auth_token=self.token       )  ,      client_kwargs={'trust_env': True}  ,      ).open()
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      **_lowercase       )						->  Tuple:
							self._get_dirs()
							lowercase_      :   str        =			self._strip_protocol(_lowercase       )
							if path in self.dir_cache:
													return self.dir_cache[path]
							else:
													raise FileNotFoundError(_lowercase       )







	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase=False  ,      **_lowercase       )						->  List[str]:
							self._get_dirs()
							lowercase_      :   List[str]        =			PurePosixPath(path.strip('/'       )       )
							lowercase_      :   List[str]        =			{}
							for p, f in self.dir_cache.items():
													lowercase_      :   Tuple        =			PurePosixPath(p.strip('/'       )       )
													lowercase_      :   Optional[int]        =			p.parent
													if root == path:
																			lowercase_      :   List[str]        =			f
							lowercase_      :   List[str]        =			list(paths.values()       )
							if detail:
													return out
							else:
													return sorted(f['name'] for f in out       )
