






import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad



class 							SCREAMING_SNAKE_CASE__		(       unittest.TestCase		):





					"""simple docstring"""






					def  _lowercase  (      self  :						Tuple ):
						snake_case__				:   Dict     =						1_0



					def  _lowercase  (      self  :						int ):
						snake_case__				:   str     =						[1, 2, 3, 4]
						snake_case__				:   Any     =						[1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
						self.assertEqual(truncate_or_pad(__A							,	self.block_size							,	0 )							,	__A )



					def  _lowercase  (      self  :						Dict ):
						snake_case__				:   Optional[int]     =						[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
						snake_case__				:   Optional[Any]     =						[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
						self.assertEqual(truncate_or_pad(__A							,	self.block_size							,	0 )							,	__A )



					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   str     =						[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0, 1_1, 1_2, 1_3]
						snake_case__				:   int     =						[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
						self.assertEqual(truncate_or_pad(__A							,	self.block_size							,	0 )							,	__A )



					def  _lowercase  (      self  :						Union[str, Any] ):
						snake_case__				:   List[str]     =						"It was the year of Our Lord one thousand seven hundred and\n        seventy-five.\n\nSpiritual revelations were conceded to England at that\n        favoured period, as at this."
						snake_case__,		snake_case__				:   Any     =						process_story(__A )
						self.assertEqual(__A							,	[] )



					def  _lowercase  (      self  :						List[str] ):
						snake_case__				:   Any     =						""
						snake_case__,		snake_case__				:   Tuple     =						process_story(__A )
						self.assertEqual(__A							,	[] )
						self.assertEqual(__A							,	[] )



					def  _lowercase  (      self  :						Dict ):
						snake_case__				:   int     =						(
						    "It was the year of Our Lord one thousand seven hundred and "
						    "seventy-five\n\nSpiritual revelations were conceded to England "
						    "at that favoured period, as at this.\n@highlight\n\nIt was the best of times"
						)
						snake_case__,		snake_case__				:   Tuple     =						process_story(__A )

						snake_case__				:   Optional[Any]     =						[
						    "It was the year of Our Lord one thousand seven hundred and seventy-five.",
						    "Spiritual revelations were conceded to England at that favoured period, as at this.",
						]
						self.assertEqual(__A							,	__A )

						snake_case__				:   Optional[Any]     =						["It was the best of times."]
						self.assertEqual(__A							,	__A )



					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   Tuple     =						torch.tensor([1, 2, 3, 4] )
						snake_case__				:   Optional[Any]     =						torch.tensor([1, 1, 1, 1] )
						np.testing.assert_array_equal(build_mask(__A							,	0 ).numpy()							,	expected.numpy() )



					def  _lowercase  (      self  :						List[str] ):
						snake_case__				:   Tuple     =						torch.tensor([1, 2, 3, 4, 2_3, 2_3, 2_3] )
						snake_case__				:   List[Any]     =						torch.tensor([1, 1, 1, 1, 0, 0, 0] )
						np.testing.assert_array_equal(build_mask(__A							,	2_3 ).numpy()							,	expected.numpy() )



					def  _lowercase  (      self  :						List[Any] ):
						snake_case__				:   List[Any]     =						torch.tensor([8, 2, 3, 4, 1, 1, 1] )
						snake_case__				:   Optional[Any]     =						torch.tensor([1, 1, 1, 1, 0, 0, 0] )
						np.testing.assert_array_equal(build_mask(__A							,	1 ).numpy()							,	expected.numpy() )






					def  _lowercase  (      self  :						Optional[Any] ):
						snake_case__				:   Union[str, Any]     =						1_0_1
						snake_case__				:   List[Any]     =						torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 1_0_1, 5, 6], [1, 1_0_1, 3, 4, 1_0_1, 6]] )
						snake_case__				:   Union[str, Any]     =						torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]] )

						snake_case__				:   Union[str, Any]     =						compute_token_type_ids(__A							,	__A )
						np.testing.assert_array_equal(__A							,	__A )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowerCamelCase      :    List[str]   		=   {"""configuration_xlnet""": ["""XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XLNetConfig"""]}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    str   		=   ["""XLNetTokenizer"""]

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    Dict   		=   ["""XLNetTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    str   		=   [
	    """XLNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XLNetForMultipleChoice""",
	    """XLNetForQuestionAnswering""",
	    """XLNetForQuestionAnsweringSimple""",
	    """XLNetForSequenceClassification""",
	    """XLNetForTokenClassification""",
	    """XLNetLMHeadModel""",
	    """XLNetModel""",
	    """XLNetPreTrainedModel""",
	    """load_tf_weights_in_xlnet""",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__lowerCamelCase      :    Dict   		=   [
	    """TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """TFXLNetForMultipleChoice""",
	    """TFXLNetForQuestionAnsweringSimple""",
	    """TFXLNetForSequenceClassification""",
	    """TFXLNetForTokenClassification""",
	    """TFXLNetLMHeadModel""",
	    """TFXLNetMainLayer""",
	    """TFXLNetModel""",
	    """TFXLNetPreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xlnet import XLNetTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_xlnet_fast import XLNetTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xlnet import (
		    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XLNetForMultipleChoice,
		    XLNetForQuestionAnswering,
		    XLNetForQuestionAnsweringSimple,
		    XLNetForSequenceClassification,
		    XLNetForTokenClassification,
		    XLNetLMHeadModel,
		    XLNetModel,
		    XLNetPreTrainedModel,
		    load_tf_weights_in_xlnet,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_xlnet import (
		    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXLNetForMultipleChoice,
		    TFXLNetForQuestionAnsweringSimple,
		    TFXLNetForSequenceClassification,
		    TFXLNetForTokenClassification,
		    TFXLNetLMHeadModel,
		    TFXLNetMainLayer,
		    TFXLNetModel,
		    TFXLNetPreTrainedModel,
		)

else:
	import sys

	__lowerCamelCase      :    int   		=   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



