



"""simple docstring"""





import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


__lowerCAmelCase       :					Any      					=					get_logger(__name__)



class    _lowerCAmelCase    (				enum.Enum		):





  """simple docstring"""






  _lowerCamelCase			  =       '''all_checks'''
  _lowerCamelCase			  =       '''basic_checks'''
  _lowerCamelCase			  =       '''no_checks'''




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[dict]					, __UpperCamelCase      :			dict					, __UpperCamelCase      :			Union[str, Any]=None    ):


      '''simple docstring'''






      if expected_checksums is None:
            logger.info("""Unable to verify checksums."""    )
            return
      if len(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    ) > 0:
            raise ExpectedMoreDownloadedFiles(str(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    )    )
      if len(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    ) > 0:
            raise UnexpectedDownloadedFile(str(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    )    )
      snake_case_   : Optional[Any]						=			[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
      snake_case_   : Optional[int]						=			""" for """ + verification_name if verification_name is not None else """"""
      if len(UpperCAmelCase__    ) > 0:
            raise NonMatchingChecksumError(
                F'Checksums didn\'t match{for_verification_name}:\n'
                F'{bad_urls}\n'
                """Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"""    )
      logger.info("""All the checksums matched successfully""" + for_verification_name    )



class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




class    _lowerCAmelCase    (				UpperCAmelCase__		):





  """simple docstring"""






  pass




def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[dict]					, __UpperCamelCase      :			dict    ):


      '''simple docstring'''






      if expected_splits is None:
            logger.info("""Unable to verify splits sizes."""    )
            return
      if len(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    ) > 0:
            raise ExpectedMoreSplits(str(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    )    )
      if len(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    ) > 0:
            raise UnexpectedSplits(str(set(UpperCAmelCase__    ) - set(UpperCAmelCase__    )    )    )
      snake_case_   : Any						=			[
          {"""expected""": expected_splits[name], """recorded""": recorded_splits[name]}
          for name in expected_splits
          if expected_splits[name].num_examples != recorded_splits[name].num_examples
      ]
      if len(UpperCAmelCase__    ) > 0:
            raise NonMatchingSplitsSizesError(str(UpperCAmelCase__    )    )
      logger.info("""All the splits matched successfully."""    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			str					, __UpperCamelCase      :			bool = True    ):


      '''simple docstring'''






      if record_checksum:
            snake_case_   : Tuple						=			shaaaa()
            with open(UpperCAmelCase__					, """rb"""    ) as f:
                  for chunk in iter(lambda: f.read(1 << 2_0    )					, B""""""    ):
                        m.update(UpperCAmelCase__    )
                  snake_case_   : Optional[int]						=			m.hexdigest()
      else:
            snake_case_   : Dict						=			None
      return {"num_bytes": os.path.getsize(UpperCAmelCase__    ), "checksum": checksum}






def 	__lowerCAmelCase						(		__UpperCamelCase      :			Tuple    ):


      '''simple docstring'''






      if dataset_size and config.IN_MEMORY_MAX_SIZE:
            return dataset_size < config.IN_MEMORY_MAX_SIZE
      else:
            return False






"""simple docstring"""





from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)

