



'''simple docstring'''



import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase



def   __lowerCAmelCase					(							lowerCamelCase   :						Optional[int]				,		lowerCamelCase   :						Optional[Any]				,		lowerCamelCase   :						Dict				,		lowerCamelCase   :						Tuple					):






	'''simple docstring'''




	if isinstance(lowerCamelCase				,		lowerCamelCase					):
		__lowerCAmelCase												=  np.full((len(lowerCamelCase					), sequence_length, 2)				,		lowerCamelCase					)
	else:
		__lowerCAmelCase												=  np.full((len(lowerCamelCase					), sequence_length)				,		lowerCamelCase					)

	for i, tensor in enumerate(lowerCamelCase					):
		if padding_side == "right":
			if isinstance(lowerCamelCase				,		lowerCamelCase					):
				__lowerCAmelCase												=  tensor[:sequence_length]
			else:
				__lowerCAmelCase												=  tensor[:sequence_length]
		else:
			if isinstance(lowerCamelCase				,		lowerCamelCase					):
				__lowerCAmelCase												=  tensor[:sequence_length]
			else:
				__lowerCAmelCase												=  tensor[:sequence_length]

	return out_tensor.tolist()



def   __lowerCAmelCase					(							lowerCamelCase   :						int					):






	'''simple docstring'''




	__lowerCAmelCase												=  ord(lowerCamelCase					)
	if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 1_23 and cp <= 1_26):
		return True
	__lowerCAmelCase												=  unicodedata.category(lowerCamelCase					)
	if cat.startswith("P"					):
		return True
	return False






@dataclass
class 					UpperCAmelCase__	(		UpperCamelCase__ ):
					a      :     PreTrainedTokenizerBase
					a      :     Union[bool, str, PaddingStrategy]        =							True
					a      :     Optional[int]        =							None
					a      :     Optional[int]        =							None
					a      :     int        =							-1_0_0
					a      :     str        =							"pt"



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			List[str]:
						import torch

						__lowerCAmelCase												=  "label" if "label" in features[0].keys() else "labels"
						__lowerCAmelCase												=  [feature[label_name] for feature in features] if label_name in features[0].keys() else None
						__lowerCAmelCase												=  self.tokenizer.pad(
						    UpperCamelCase			,				padding=self.padding			,				max_length=self.max_length			,				pad_to_multiple_of=self.pad_to_multiple_of			,				return_tensors="pt" if labels is None else None			,				)

						if labels is None:
							return batch

						__lowerCAmelCase												=  torch.tensor(batch["entity_ids"]     ).shape[1]
						__lowerCAmelCase												=  self.tokenizer.padding_side
						if padding_side == "right":
							__lowerCAmelCase												=  [
							    list(UpperCamelCase     ) + [self.label_pad_token_id] * (sequence_length - len(UpperCamelCase     )) for label in labels
							]
						else:
							__lowerCAmelCase												=  [
							    [self.label_pad_token_id] * (sequence_length - len(UpperCamelCase     )) + list(UpperCamelCase     ) for label in labels
							]

						__lowerCAmelCase												=  [feature["ner_tags"] for feature in features]
						__lowerCAmelCase												=  padding_tensor(UpperCamelCase			,				-1			,				UpperCamelCase			,				UpperCamelCase     )
						__lowerCAmelCase												=  [feature["original_entity_spans"] for feature in features]
						__lowerCAmelCase												=  padding_tensor(UpperCamelCase			,				(-1, -1)			,				UpperCamelCase			,				UpperCamelCase     )
						__lowerCAmelCase												=  {k: torch.tensor(UpperCamelCase			,				dtype=torch.intaa     ) for k, v in batch.items()}

						return batch




'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase	:		Union[str, Any]				=   {
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Tuple				=   ['''DistilBertTokenizerFast''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Tuple				=   [
	    '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''DistilBertForMaskedLM''',
	    '''DistilBertForMultipleChoice''',
	    '''DistilBertForQuestionAnswering''',
	    '''DistilBertForSequenceClassification''',
	    '''DistilBertForTokenClassification''',
	    '''DistilBertModel''',
	    '''DistilBertPreTrainedModel''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		List[str]				=   [
	    '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFDistilBertForMaskedLM''',
	    '''TFDistilBertForMultipleChoice''',
	    '''TFDistilBertForQuestionAnswering''',
	    '''TFDistilBertForSequenceClassification''',
	    '''TFDistilBertForTokenClassification''',
	    '''TFDistilBertMainLayer''',
	    '''TFDistilBertModel''',
	    '''TFDistilBertPreTrainedModel''',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	lowerCAmelCase	:		Optional[int]				=   [
	    '''FlaxDistilBertForMaskedLM''',
	    '''FlaxDistilBertForMultipleChoice''',
	    '''FlaxDistilBertForQuestionAnswering''',
	    '''FlaxDistilBertForSequenceClassification''',
	    '''FlaxDistilBertForTokenClassification''',
	    '''FlaxDistilBertModel''',
	    '''FlaxDistilBertPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_distilbert import (
	    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    DistilBertConfig,
	    DistilBertOnnxConfig,
	)
	from .tokenization_distilbert import DistilBertTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_distilbert_fast import DistilBertTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_distilbert import (
		    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DistilBertForMaskedLM,
		    DistilBertForMultipleChoice,
		    DistilBertForQuestionAnswering,
		    DistilBertForSequenceClassification,
		    DistilBertForTokenClassification,
		    DistilBertModel,
		    DistilBertPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_distilbert import (
		    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFDistilBertForMaskedLM,
		    TFDistilBertForMultipleChoice,
		    TFDistilBertForQuestionAnswering,
		    TFDistilBertForSequenceClassification,
		    TFDistilBertForTokenClassification,
		    TFDistilBertMainLayer,
		    TFDistilBertModel,
		    TFDistilBertPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_distilbert import (
		    FlaxDistilBertForMaskedLM,
		    FlaxDistilBertForMultipleChoice,
		    FlaxDistilBertForQuestionAnswering,
		    FlaxDistilBertForSequenceClassification,
		    FlaxDistilBertForTokenClassification,
		    FlaxDistilBertModel,
		    FlaxDistilBertPreTrainedModel,
		)

else:
	import sys

	lowerCAmelCase	:		List[str]				=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)