




import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							def __init__(					self       :	Dict       ,     a       :	Union[str, Any]       ,     a       :	Optional[int]			)			->					List[str]:
										SCREAMING_SNAKE_CASE           =					params

										SCREAMING_SNAKE_CASE           =					np.array(a			)
										SCREAMING_SNAKE_CASE           =					np.array([len(a			) for t in data]			)

										self.check()
										self.remove_long_sequences()
										self.remove_empty_sequences()
										self.remove_unknown_sequences()
										self.check()
										self.print_statistics()





							def __getitem__(					self       :	str       ,     a       :	Optional[Any]			)			->					List[Any]:
										return (self.token_ids[index], self.lengths[index])





							def __len__(					self       :	Dict			)			->					Any:
										return len(self.lengths			)





							def    _UpperCAmelCase     (					self       :	str			)			->					Union[str, Any]:
										assert len(self.token_ids			) == len(self.lengths			)
										assert all(self.lengths[i] == len(self.token_ids[i]			) for i in range(len(self.lengths			)			)			)





							def    _UpperCAmelCase     (					self       :	str			)			->					List[Any]:
										SCREAMING_SNAKE_CASE           =					self.params.max_model_input_size
										SCREAMING_SNAKE_CASE           =					self.lengths > max_len
										logger.info(f"""Splitting {sum(a			)} too long sequences."""			)

										def divide_chunks(a       :	Optional[Any]       ,     a       :	Union[str, Any]			):
													return [l[i : i + n] for i in range(0       ,     len(a			)       ,     a			)]

										SCREAMING_SNAKE_CASE           =					[]
										SCREAMING_SNAKE_CASE           =					[]
										if self.params.mlm:
													SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					self.params.special_tok_ids["""cls_token"""], self.params.special_tok_ids["""sep_token"""]
										else:
													SCREAMING_SNAKE_CASE     ,  SCREAMING_SNAKE_CASE           =					self.params.special_tok_ids["""bos_token"""], self.params.special_tok_ids["""eos_token"""]

										for seq_, len_ in zip(self.token_ids       ,     self.lengths			):
													assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
													if len_ <= max_len:
																new_tok_ids.append(seq_			)
																new_lengths.append(len_			)
													else:
																SCREAMING_SNAKE_CASE           =					[]
																for sub_s in divide_chunks(seq_       ,     max_len - 2			):
																			if sub_s[0] != cls_id:
																						SCREAMING_SNAKE_CASE           =					np.insert(a       ,     0       ,     a			)
																			if sub_s[-1] != sep_id:
																						SCREAMING_SNAKE_CASE           =					np.insert(a       ,     len(a			)       ,     a			)
																			assert len(a			) <= max_len
																			assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																			sub_seqs.append(a			)

																new_tok_ids.extend(a			)
																new_lengths.extend([len(a			) for l in sub_seqs]			)

										SCREAMING_SNAKE_CASE           =					np.array(a			)
										SCREAMING_SNAKE_CASE           =					np.array(a			)





							def    _UpperCAmelCase     (					self       :	Dict			)			->					List[Any]:
										SCREAMING_SNAKE_CASE           =					len(self			)
										SCREAMING_SNAKE_CASE           =					self.lengths > 11
										SCREAMING_SNAKE_CASE           =					self.token_ids[indices]
										SCREAMING_SNAKE_CASE           =					self.lengths[indices]
										SCREAMING_SNAKE_CASE           =					len(self			)
										logger.info(f"""Remove {init_size - new_size} too short (<=11 tokens) sequences."""			)





							def    _UpperCAmelCase     (					self       :	Any			)			->					int:
										if "unk_token" not in self.params.special_tok_ids:
													return
										else:
													SCREAMING_SNAKE_CASE           =					self.params.special_tok_ids["""unk_token"""]
										SCREAMING_SNAKE_CASE           =					len(self			)
										SCREAMING_SNAKE_CASE           =					np.array([np.count_nonzero(a == unk_token_id			) for a in self.token_ids]			)
										SCREAMING_SNAKE_CASE           =					(unk_occs / self.lengths) < 0.5
										SCREAMING_SNAKE_CASE           =					self.token_ids[indices]
										SCREAMING_SNAKE_CASE           =					self.lengths[indices]
										SCREAMING_SNAKE_CASE           =					len(self			)
										logger.info(f"""Remove {init_size - new_size} sequences with a high level of unknown tokens (50%)."""			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Dict:
										if not self.params.is_master:
													return
										logger.info(f"""{len(self			)} sequences"""			)
										# data_len = sum(self.lengths)
										# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
										# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

										# unk_idx = self.params.special_tok_ids['unk_token']
										# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
										# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')







							def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	int			)			->					Dict:
										SCREAMING_SNAKE_CASE           =					[t[0] for t in batch]
										SCREAMING_SNAKE_CASE           =					[t[1] for t in batch]
										assert len(a			) == len(a			)

										# Max for paddings
										SCREAMING_SNAKE_CASE           =					max(a			)

										# Pad token ids
										if self.params.mlm:
													SCREAMING_SNAKE_CASE           =					self.params.special_tok_ids["""pad_token"""]
										else:
													SCREAMING_SNAKE_CASE           =					self.params.special_tok_ids["""unk_token"""]
										SCREAMING_SNAKE_CASE           =					[list(t.astype(a			)			) + [pad_idx] * (max_seq_len_ - len(a			)) for t in token_ids]
										assert len(tk_			) == len(a			)
										assert all(len(a			) == max_seq_len_ for t in tk_			)

										SCREAMING_SNAKE_CASE           =					torch.tensor(tk_			)  # (bs, max_seq_len_)
										SCREAMING_SNAKE_CASE           =					torch.tensor(a			)  # (bs)
										return tk_t, lg_t












import os
import sys
import unittest


__A						:							Any							=  os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
__A						:							List[Any]							=  os.path.join(git_repo_path, """src""", """diffusers""")




class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					find_backend("""    if not is_torch_available():"""			)
										self.assertEqual(a       ,     """torch"""			)

										# backend_with_underscore = find_backend("    if not is_tensorflow_text_available():")
										# self.assertEqual(backend_with_underscore, "tensorflow_text")

										SCREAMING_SNAKE_CASE           =					find_backend("""    if not (is_torch_available() and is_transformers_available()):"""			)
										self.assertEqual(a       ,     """torch_and_transformers"""			)

										# double_backend_with_underscore = find_backend(
										#    "    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
										# )
										# self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")

										SCREAMING_SNAKE_CASE           =					find_backend(
										    """    if not (is_torch_available() and is_transformers_available() and is_onnx_available()):"""			)
										self.assertEqual(a       ,     """torch_and_transformers_and_onnx"""			)





							def    _UpperCAmelCase     (					self       :	List[str]			)			->					int:
										SCREAMING_SNAKE_CASE           =					read_init()
										# We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
										self.assertIn("""torch"""       ,     a			)
										self.assertIn("""torch_and_transformers"""       ,     a			)
										self.assertIn("""flax_and_transformers"""       ,     a			)
										self.assertIn("""torch_and_transformers_and_onnx"""       ,     a			)

										# Likewise, we can't assert on the exact content of a key
										self.assertIn("""UNet2DModel"""       ,     objects["""torch"""]			)
										self.assertIn("""FlaxUNet2DConditionModel"""       ,     objects["""flax"""]			)
										self.assertIn("""StableDiffusionPipeline"""       ,     objects["""torch_and_transformers"""]			)
										self.assertIn("""FlaxStableDiffusionPipeline"""       ,     objects["""flax_and_transformers"""]			)
										self.assertIn("""LMSDiscreteScheduler"""       ,     objects["""torch_and_scipy"""]			)
										self.assertIn("""OnnxStableDiffusionPipeline"""       ,     objects["""torch_and_transformers_and_onnx"""]			)





							def    _UpperCAmelCase     (					self       :	Any			)			->					int:
										SCREAMING_SNAKE_CASE           =					create_dummy_object("""CONSTANT"""       ,     """'torch'"""			)
										self.assertEqual(a       ,     """\nCONSTANT = None\n"""			)

										SCREAMING_SNAKE_CASE           =					create_dummy_object("""function"""       ,     """'torch'"""			)
										self.assertEqual(
										    a       ,     """\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n"""			)

										SCREAMING_SNAKE_CASE           =					"""
class FakeClass(metaclass=DummyObject):
    _backends = 'torch'

    def __init__(self, *args, **kwargs):
        requires_backends(self, 'torch')

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, 'torch')

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, 'torch')
"""
										SCREAMING_SNAKE_CASE           =					create_dummy_object("""FakeClass"""       ,     """'torch'"""			)
										self.assertEqual(a       ,     a			)







							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Union[str, Any]:
										SCREAMING_SNAKE_CASE           =					"""# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, [\"torch\"])


class FakeClass(metaclass=DummyObject):
    _backends = [\"torch\"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, [\"torch\"])

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, [\"torch\"])

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, [\"torch\"])
"""
										SCREAMING_SNAKE_CASE           =					create_dummy_files({"""torch""": ["""CONSTANT""", """function""", """FakeClass"""]}			)
										self.assertEqual(dummy_files["""torch"""]       ,     a			)






