





'''simple docstring'''

import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


UpperCAmelCase					:		List[str]		=							'\\n    Text data.\n    Second line of data.'

UpperCAmelCase					:		Dict		=							'file'




@pytest.fixture(scope='''session'''	)
def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Dict					     =     tmp_path_factory.mktemp('''data'''	) / (FILE_PATH + '''.zstd''')
      _snake_case		:	Dict					     =     bytes(lowerCAmelCase_			,      '''utf-8'''	)
      with zstd.open(lowerCAmelCase_			,      '''wb'''	) as f:
            f.write(lowerCAmelCase_	)
      return path




@pytest.fixture
def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      with open(os.path.join(tmpfs.local_root_dir			,      lowerCAmelCase_	)			,      '''w'''	) as f:
            f.write(lowerCAmelCase_	)
      return FILE_PATH




@pytest.mark.parametrize('''compression_format'''			,      ['''gzip''', '''xz''', '''zstd''']	)
def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	List[Any]					     =     {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
      _snake_case		:	Optional[Any]					     =     input_paths[compression_format]
      _snake_case		:	str					     =     tmp_path / '''cache'''
      _snake_case		:	int					     =     DownloadConfig(cache_dir=lowerCAmelCase_			,      extract_compressed_file=lowerCAmelCase_	)
      _snake_case		:	Optional[Any]					     =     cached_path(lowerCAmelCase_			,      download_config=lowerCAmelCase_	)
      with open(lowerCAmelCase_	) as f:
            _snake_case		:	List[str]					     =     f.read()
      with open(lowerCAmelCase_	) as f:
            _snake_case		:	str					     =     f.read()
      assert extracted_file_content == expected_file_content




@pytest.mark.parametrize('''default_extracted'''			,      [True, False]	)
@pytest.mark.parametrize('''default_cache_dir'''			,      [True, False]	)
def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Union[str, Any]					     =     '''custom_cache'''
      _snake_case		:	int					     =     '''custom_extracted_dir'''
      _snake_case		:	Dict					     =     tmp_path / '''custom_extracted_path'''
      if default_extracted:
            _snake_case		:	Dict					     =     ('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
      else:
            monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''			,      lowerCAmelCase_	)
            monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''			,      str(lowerCAmelCase_	)	)
            _snake_case		:	List[Any]					     =     custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

      _snake_case		:	Optional[int]					     =     xz_file
      _snake_case		:	Union[str, Any]					     =     (
          DownloadConfig(extract_compressed_file=lowerCAmelCase_	)
          if default_cache_dir
          else DownloadConfig(cache_dir=tmp_path / custom_cache_dir			,      extract_compressed_file=lowerCAmelCase_	)
      )
      _snake_case		:	str					     =     cached_path(lowerCAmelCase_			,      download_config=lowerCAmelCase_	)
      assert Path(lowerCAmelCase_	).parent.parts[-2:] == expected




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Dict					     =     str(Path(lowerCAmelCase_	).resolve()	)
      assert cached_path(lowerCAmelCase_	) == text_file
      # relative path
      _snake_case		:	Optional[int]					     =     str(Path(lowerCAmelCase_	).resolve().relative_to(Path(os.getcwd()	)	)	)
      assert cached_path(lowerCAmelCase_	) == text_file




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	str					     =     str(tmp_path.resolve() / '''__missing_file__.txt'''	)
      with pytest.raises(lowerCAmelCase_	):
            cached_path(lowerCAmelCase_	)
      # relative path
      _snake_case		:	Optional[Any]					     =     '''./__missing_file__.txt'''
      with pytest.raises(lowerCAmelCase_	):
            cached_path(lowerCAmelCase_	)




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	str					     =     get_from_cache(f'''tmp://{tmpfs_file}'''	)
      with open(lowerCAmelCase_	) as f:
            _snake_case		:	Dict					     =     f.read()
      assert output_file_content == FILE_CONTENT




@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,      lowerCAmelCase_	)
def    _a    (							):





      """simple docstring"""



      with pytest.raises(lowerCAmelCase_	):
            cached_path('''https://huggingface.co'''	)




@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,      lowerCAmelCase_	)
def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Union[str, Any]					     =     tmp_path_factory.mktemp('''data'''	) / '''file.html'''
      with pytest.raises(lowerCAmelCase_	):
            http_get('''https://huggingface.co'''			,      temp_file=lowerCAmelCase_	)
      with pytest.raises(lowerCAmelCase_	):
            http_head('''https://huggingface.co'''	)




@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,      lowerCAmelCase_	)
def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Optional[Any]					     =     tmp_path_factory.mktemp('''data'''	) / '''file.html'''
      with pytest.raises(lowerCAmelCase_	):
            ftp_get('''ftp://huggingface.co'''			,      temp_file=lowerCAmelCase_	)
      with pytest.raises(lowerCAmelCase_	):
            ftp_head('''ftp://huggingface.co'''	)




@patch('''datasets.config.HF_DATASETS_OFFLINE'''			,      lowerCAmelCase_	)
def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Optional[Any]					     =     tmp_path_factory.mktemp('''data'''	) / '''file.html'''
      with pytest.raises(lowerCAmelCase_	):
            fsspec_get('''s3://huggingface.co'''			,      temp_file=lowerCAmelCase_	)
      with pytest.raises(lowerCAmelCase_	):
            fsspec_head('''s3://huggingface.co'''	)









'''simple docstring'''

import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
   import optax
   from flax.training.common_utils import onehot

   from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
   from transformers.models.ta.modeling_flax_ta import shift_tokens_right





@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class lowerCamelCase						(unittest.TestCase					):







  @slow
  def 				UpperCAmelCase_				(		self		)		->						int:






        """simple docstring"""





        _snake_case		:	Tuple					     =     FlaxMTaForConditionalGeneration.from_pretrained('''google/mt5-small'''		)
        _snake_case		:	Any					     =     AutoTokenizer.from_pretrained('''google/mt5-small'''		)

        _snake_case		:	List[str]					     =     tokenizer('''Hello there'''     ,     return_tensors='''np'''		).input_ids
        _snake_case		:	Dict					     =     tokenizer('''Hi I am'''     ,     return_tensors='''np'''		).input_ids

        _snake_case		:	Any					     =     shift_tokens_right(lowercase__     ,     model.config.pad_token_id     ,     model.config.decoder_start_token_id		)

        _snake_case		:	Any					     =     model(lowercase__     ,     decoder_input_ids=lowercase__		).logits
        _snake_case		:	Tuple					     =     optax.softmax_cross_entropy(lowercase__     ,     onehot(lowercase__     ,     logits.shape[-1]		)		).mean()

        _snake_case		:	Tuple					     =     -(labels.shape[-1] * loss.item())

        _snake_case		:	Union[str, Any]					     =     -84.9_127
        self.assertTrue(abs(mtf_score - EXPECTED_SCORE		) < 1E-4		)


