



"""simple docstring"""

import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

lowercase__			   =  "python tqdm regex requests packaging filelock numpy tokenizers".split()
if sys.version_info < (3, 7):
			pkgs_to_check_at_runtime.append("dataclasses")
if sys.version_info < (3, 8):
			pkgs_to_check_at_runtime.append("importlib_metadata")

for pkg in pkgs_to_check_at_runtime:
			if pkg in deps:
						if pkg == "tokenizers":
									# must be loaded here, or else tqdm check may fail
									from .utils import is_tokenizers_available

									if not is_tokenizers_available():
												continue  # not required, check version only if installed

						require_version_core(deps[pkg])
			else:
						raise ValueError(f'can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py')





def       __magic_name__    (   _lowerCamelCase				: Optional[int]  ,       _lowerCamelCase				: str=None					):
							require_version(deps[pkg]  ,       _lowerCamelCase					)






"""simple docstring"""

from unittest import TestCase

from datasets import Dataset
from minhash_deduplication import deduplicate_dataset, make_duplicate_clusters





def       __magic_name__    (   ):
							__a		:		Dict									=       {
							    """repo_name""": ["""test_repo1""", """test_repo2""", """test_repo3"""],
							    """path""": ["""test_1.py""", """test_2.py""", """unit_test.py"""],
							    """content""": ["""a """ * 2_0, """a """ * 3_0, """b """ * 7],
							}
							__a		:		Optional[Any]									=       Dataset.from_dict(_lowerCamelCase					)
							return dataset







class SCREAMING_SNAKE_CASE__					(		__snake_case      ):



						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Optional[int]									=       get_dataset()
													__a		:		List[Any]									=       make_duplicate_clusters(_lowercase						,     0.85							)
													self.assertEqual(len(duplicate_clusters[0]							)						,     2							)





						def     lowerCAmelCase__(self							):





													'''simple docstring'''

													__a		:		Tuple									=       get_dataset()
													__a							,     __a		:		Optional[Any]									=       deduplicate_dataset(_lowercase							)
													self.assertEqual(len(_lowercase							)						,     2							)
													print(_lowercase							)
													self.assertEqual(duplicate_clusters[0][0]["""copies"""]						,     2							)
													self.assertEqual(duplicate_clusters[0][0]["""is_extreme"""]						,     _lowercase							)

