



"""simple docstring"""
import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
   import PIL.Image

   from .features import FeatureType


_UpperCAmelCase     =			None
_UpperCAmelCase     =			"""<""" if sys.byteorder == """little""" else """>"""
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
_UpperCAmelCase     =			[
    np.dtype("""|b1"""),
    np.dtype("""|u1"""),
    np.dtype("""<u2"""),
    np.dtype(""">u2"""),
    np.dtype("""<i2"""),
    np.dtype(""">i2"""),
    np.dtype("""<u4"""),
    np.dtype(""">u4"""),
    np.dtype("""<i4"""),
    np.dtype(""">i4"""),
    np.dtype("""<f4"""),
    np.dtype(""">f4"""),
    np.dtype("""<f8"""),
    np.dtype(""">f8"""),
]
@dataclass
class 					a	:
  UpperCamelCase	:				bool     						=					True
  UpperCamelCase	:				Optional[str]     						=					None
  # Automatically constructed
  UpperCamelCase	:				ClassVar[str]     						=					"PIL.Image.Image"
  UpperCamelCase	:				ClassVar[Any]     						=					pa.struct({'bytes': pa.binary(), 'path': pa.string()}       )
  UpperCamelCase	:				str     						=					field(default='Image'	,      init=UpperCAmelCase__	,      repr=UpperCAmelCase__       )



  def __call__(						self :					List[str]		)     ->					Tuple:


     '''simple docstring'''




     return self.pa_type



  def 	lowerCamelCase__       (						self :					List[str]       ,		lowerCAmelCase :					Union[str, bytes, dict, np.ndarray, "PIL.Image.Image"]		)     ->					dict:


     '''simple docstring'''




     if config.PIL_AVAILABLE:
        import PIL.Image
     else:
        raise ImportError("""To support encoding images, please install 'Pillow'."""		)

     if isinstance(lowerCAmelCase       ,		lowerCAmelCase		):
        SCREAMING_SNAKE_CASE_: Union[str, Any]     			=np.array(lowerCAmelCase		)

     if isinstance(lowerCAmelCase       ,		lowerCAmelCase		):
        return {"path": value, "bytes": None}
     elif isinstance(lowerCAmelCase       ,		lowerCAmelCase		):
        return {"path": None, "bytes": value}
     elif isinstance(lowerCAmelCase       ,		np.ndarray		):
        # convert the image array to PNG/TIFF bytes
        return encode_np_array(lowerCAmelCase		)
     elif isinstance(lowerCAmelCase       ,		PIL.Image.Image		):
        # convert the PIL image to bytes (default format is PNG/TIFF)
        return encode_pil_image(lowerCAmelCase		)
     elif value.get("""path"""		) is not None and os.path.isfile(value["""path"""]		):
        # we set "bytes": None to not duplicate the data if they're already available locally
        return {"bytes": None, "path": value.get("""path"""		)}
     elif value.get("""bytes"""		) is not None or value.get("""path"""		) is not None:
        # store the image bytes, and path is used to infer the image format using the file extension
        return {"bytes": value.get("""bytes"""		), "path": value.get("""path"""		)}
     else:
        raise ValueError(
            f'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''		)



  def 	lowerCamelCase__       (						self :					Optional[Any]       ,		lowerCAmelCase :					dict       ,		lowerCAmelCase :					Tuple=None		)     ->					"PIL.Image.Image":


     '''simple docstring'''




     if not self.decode:
        raise RuntimeError("""Decoding is disabled for this feature. Please use Image(decode=True) instead."""		)

     if config.PIL_AVAILABLE:
        import PIL.Image
     else:
        raise ImportError("""To support decoding images, please install 'Pillow'."""		)

     if token_per_repo_id is None:
        SCREAMING_SNAKE_CASE_: int     			={}

     SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: Optional[Any]     			=value["""path"""], value["""bytes"""]
     if bytes_ is None:
        if path is None:
           raise ValueError(f'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''		)
        else:
           if is_local_path(lowerCAmelCase		):
              SCREAMING_SNAKE_CASE_: int     			=PIL.Image.open(lowerCAmelCase		)
           else:
              SCREAMING_SNAKE_CASE_: int     			=path.split("""::"""		)[-1]
              try:
                 SCREAMING_SNAKE_CASE_: Optional[int]     			=string_to_dict(lowerCAmelCase       ,		config.HUB_DATASETS_URL		)["""repo_id"""]
                 SCREAMING_SNAKE_CASE_: List[Any]     			=token_per_repo_id.get(lowerCAmelCase		)
              except ValueError:
                 SCREAMING_SNAKE_CASE_: List[Any]     			=None
              with xopen(lowerCAmelCase       ,		"""rb"""       ,		use_auth_token=lowerCAmelCase		) as f:
                 SCREAMING_SNAKE_CASE_: int     			=BytesIO(f.read()		)
              SCREAMING_SNAKE_CASE_: Union[str, Any]     			=PIL.Image.open(bytes_		)
     else:
        SCREAMING_SNAKE_CASE_: int     			=PIL.Image.open(BytesIO(bytes_		)		)
     image.load()  # to avoid "Too many open files" errors
     return image



  def 	lowerCamelCase__       (						self :					Tuple		)     ->					Union["FeatureType", Dict[str, "FeatureType"]]:


     '''simple docstring'''




     from .features import Value

     return (
         self
         if self.decode
         else {
             "bytes": Value("""binary"""		),
             "path": Value("""string"""		),
         }
     )



  def 	lowerCamelCase__       (						self :					Optional[int]       ,		lowerCAmelCase :					Union[pa.StringArray, pa.StructArray, pa.ListArray]		)     ->					pa.StructArray:


     '''simple docstring'''




     if pa.types.is_string(storage.type		):
        SCREAMING_SNAKE_CASE_: Dict     			=pa.array([None] * len(lowerCAmelCase		)       ,		type=pa.binary()		)
        SCREAMING_SNAKE_CASE_: int     			=pa.StructArray.from_arrays([bytes_array, storage]       ,		["""bytes""", """path"""]       ,		mask=storage.is_null()		)
     elif pa.types.is_binary(storage.type		):
        SCREAMING_SNAKE_CASE_: Tuple     			=pa.array([None] * len(lowerCAmelCase		)       ,		type=pa.string()		)
        SCREAMING_SNAKE_CASE_: Optional[Any]     			=pa.StructArray.from_arrays([storage, path_array]       ,		["""bytes""", """path"""]       ,		mask=storage.is_null()		)
     elif pa.types.is_struct(storage.type		):
        if storage.type.get_field_index("""bytes"""		) >= 0:
           SCREAMING_SNAKE_CASE_: Optional[int]     			=storage.field("""bytes"""		)
        else:
           SCREAMING_SNAKE_CASE_: str     			=pa.array([None] * len(lowerCAmelCase		)       ,		type=pa.binary()		)
        if storage.type.get_field_index("""path"""		) >= 0:
           SCREAMING_SNAKE_CASE_: List[str]     			=storage.field("""path"""		)
        else:
           SCREAMING_SNAKE_CASE_: Dict     			=pa.array([None] * len(lowerCAmelCase		)       ,		type=pa.string()		)
        SCREAMING_SNAKE_CASE_: Optional[int]     			=pa.StructArray.from_arrays([bytes_array, path_array]       ,		["""bytes""", """path"""]       ,		mask=storage.is_null()		)
     elif pa.types.is_list(storage.type		):
        SCREAMING_SNAKE_CASE_: Any     			=pa.array(
            [encode_np_array(np.array(lowerCAmelCase		)		)["""bytes"""] if arr is not None else None for arr in storage.to_pylist()]       ,		type=pa.binary()       ,		)
        SCREAMING_SNAKE_CASE_: int     			=pa.array([None] * len(lowerCAmelCase		)       ,		type=pa.string()		)
        SCREAMING_SNAKE_CASE_: Tuple     			=pa.StructArray.from_arrays(
            [bytes_array, path_array]       ,		["""bytes""", """path"""]       ,		mask=bytes_array.is_null()		)
     return array_cast(lowerCAmelCase       ,		self.pa_type		)





  def 	lowerCamelCase__       (						self :					List[str]       ,		lowerCAmelCase :					pa.StructArray		)     ->					pa.StructArray:


     '''simple docstring'''




     @no_op_if_value_is_null
     def path_to_bytes(lowerCAmelCase :					List[str]		):
        with xopen(lowerCAmelCase       ,		"""rb"""		) as f:
           SCREAMING_SNAKE_CASE_: List[Any]     			=f.read()
        return bytes_

     SCREAMING_SNAKE_CASE_: int     			=pa.array(
         [
             (path_to_bytes(x["""path"""]		) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
             for x in storage.to_pylist()
         ]       ,		type=pa.binary()       ,		)
     SCREAMING_SNAKE_CASE_: Dict     			=pa.array(
         [os.path.basename(lowerCAmelCase		) if path is not None else None for path in storage.field("""path"""		).to_pylist()]       ,		type=pa.string()       ,		)
     SCREAMING_SNAKE_CASE_: Optional[int]     			=pa.StructArray.from_arrays([bytes_array, path_array]       ,		["""bytes""", """path"""]       ,		mask=bytes_array.is_null()		)
     return array_cast(lowerCAmelCase       ,		self.pa_type		)







def      __magic_name__				(							):
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("""To support encoding images, please install 'Pillow'.""" )

   global _IMAGE_COMPRESSION_FORMATS
   if _IMAGE_COMPRESSION_FORMATS is None:
      PIL.Image.init()
      SCREAMING_SNAKE_CASE_: Any     			=list(set(PIL.Image.OPEN.keys() ) & set(PIL.Image.SAVE.keys() ) )
   return _IMAGE_COMPRESSION_FORMATS





def      __magic_name__				(							lowercase ):
   SCREAMING_SNAKE_CASE_: List[Any]     			=BytesIO()
   if image.format in list_image_compression_formats():
      SCREAMING_SNAKE_CASE_: Optional[int]     			=image.format
   else:
      SCREAMING_SNAKE_CASE_: Any     			="""PNG""" if image.mode in ["""1""", """L""", """LA""", """RGB""", """RGBA"""] else """TIFF"""
   image.save(lowercase							,			format=lowercase )
   return buffer.getvalue()





def      __magic_name__				(							lowercase ):
   if hasattr(lowercase							,			"""filename""" ) and image.filename != "":
      return {"path": image.filename, "bytes": None}
   else:
      return {"path": None, "bytes": image_to_bytes(lowercase )}





def      __magic_name__				(							lowercase ):
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("""To support encoding images, please install 'Pillow'.""" )

   SCREAMING_SNAKE_CASE_: List[Any]     			=array.dtype
   SCREAMING_SNAKE_CASE_: List[Any]     			=dtype.byteorder if dtype.byteorder != """=""" else _NATIVE_BYTEORDER
   SCREAMING_SNAKE_CASE_: int     			=dtype.kind
   SCREAMING_SNAKE_CASE_: str     			=dtype.itemsize

   SCREAMING_SNAKE_CASE_: Optional[Any]     			=None

   # Multi-channel array case (only np.dtype("|u1") is allowed)
   if array.shape[2:]:
      SCREAMING_SNAKE_CASE_: List[str]     			=np.dtype("""|u1""" )
      if dtype_kind not in ["u", "i"]:
         raise TypeError(
             f'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.''' )
      if dtype is not dest_dtype:
         warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
    # Exact match
   elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
      SCREAMING_SNAKE_CASE_: int     			=dtype
   else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
      while dtype_itemsize >= 1:
         SCREAMING_SNAKE_CASE_: Any     			=dtype_byteorder + dtype_kind + str(lowercase )
         SCREAMING_SNAKE_CASE_: List[str]     			=np.dtype(lowercase )
         if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
            warnings.warn(f'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\'''' )
            break
         else:
            dtype_itemsize //= 2

   if dest_dtype is None:
      raise TypeError(
          f'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}''' )
   SCREAMING_SNAKE_CASE_: int     			=PIL.Image.fromarray(array.astype(lowercase ) )
   return {"path": None, "bytes": image_to_bytes(lowercase )}





def      __magic_name__				(							lowercase ):
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("""To support encoding images, please install 'Pillow'.""" )

   if objs:
      SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: Union[str, Any]     			=first_non_null_value(lowercase )
      if isinstance(lowercase							,			lowercase ):
         return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
      if isinstance(lowercase							,			np.ndarray ):
         SCREAMING_SNAKE_CASE_: List[str]     			=no_op_if_value_is_null(lowercase )
         return [obj_to_image_dict_func(lowercase ) for obj in objs]
      elif isinstance(lowercase							,			PIL.Image.Image ):
         SCREAMING_SNAKE_CASE_: List[Any]     			=no_op_if_value_is_null(lowercase )
         return [obj_to_image_dict_func(lowercase ) for obj in objs]
      else:
         return objs
   else:
      return objs








"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_UpperCAmelCase     =			{
    """configuration_xlm""": ["""XLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XLMConfig""", """XLMOnnxConfig"""],
    """tokenization_xlm""": ["""XLMTokenizer"""],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _UpperCAmelCase     =			[
       """XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """XLMForMultipleChoice""",
       """XLMForQuestionAnswering""",
       """XLMForQuestionAnsweringSimple""",
       """XLMForSequenceClassification""",
       """XLMForTokenClassification""",
       """XLMModel""",
       """XLMPreTrainedModel""",
       """XLMWithLMHeadModel""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _UpperCAmelCase     =			[
       """TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """TFXLMForMultipleChoice""",
       """TFXLMForQuestionAnsweringSimple""",
       """TFXLMForSequenceClassification""",
       """TFXLMForTokenClassification""",
       """TFXLMMainLayer""",
       """TFXLMModel""",
       """TFXLMPreTrainedModel""",
       """TFXLMWithLMHeadModel""",
   ]


if TYPE_CHECKING:
   from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
   from .tokenization_xlm import XLMTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_xlm import (
          XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          XLMForMultipleChoice,
          XLMForQuestionAnswering,
          XLMForQuestionAnsweringSimple,
          XLMForSequenceClassification,
          XLMForTokenClassification,
          XLMModel,
          XLMPreTrainedModel,
          XLMWithLMHeadModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_xlm import (
          TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFXLMForMultipleChoice,
          TFXLMForQuestionAnsweringSimple,
          TFXLMForSequenceClassification,
          TFXLMForTokenClassification,
          TFXLMMainLayer,
          TFXLMModel,
          TFXLMPreTrainedModel,
          TFXLMWithLMHeadModel,
      )

else:
   import sys

   _UpperCAmelCase     =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



