



import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


lowercase_								=     logging.get_logger(__name__)

# General docstring
lowercase_								=     '''PoolFormerConfig'''

# Base docstring
lowercase_								=     '''sail/poolformer_s12'''
lowercase_								=     [1, 512, 7, 7]

# Image classification docstring
lowercase_								=     '''sail/poolformer_s12'''
lowercase_								=     '''tabby, tabby cat'''

lowercase_								=     [
    '''sail/poolformer_s12''',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_ = 0.0						,  SCREAMING_SNAKE_CASE_ = False  ):

			if drop_prob == 0.0 or not training:
						return input
			lowercase__					  =				1 - drop_prob
			lowercase__					  =				(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
			lowercase__					  =				keep_prob + torch.rand(UpperCAmelCase__						,  dtype=input.dtype						,  device=input.device  )
			random_tensor.floor_()  # binarize
			lowercase__					  =				input.div(UpperCAmelCase__  ) * random_tensor
			return output
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Tuple,					__lowercase					:						Optional[float] = None						):
							super().__init__()
							lowercase__					  =				drop_prob




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						torch.Tensor						):
							return drop_path(__lowerCAmelCase,					self.drop_prob,					self.training						)




				def 							A__      (						self					:						List[str]						):
							return "p={}".format(self.drop_prob						)
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Union[str, Any],					__lowercase					:						Tuple,					__lowercase					:						Union[str, Any],					__lowercase					:						Any,					__lowercase					:						Union[str, Any],					__lowercase					:						Dict,					__lowercase					:						Dict=None						):
							super().__init__()
							lowercase__					  =				patch_size if isinstance(__lowerCAmelCase,					collections.abc.Iterable						) else (patch_size, patch_size)
							lowercase__					  =				stride if isinstance(__lowerCAmelCase,					collections.abc.Iterable						) else (stride, stride)
							lowercase__					  =				padding if isinstance(__lowerCAmelCase,					collections.abc.Iterable						) else (padding, padding)

							lowercase__					  =				nn.Convad(__lowerCAmelCase,					__lowerCAmelCase,					kernel_size=__lowerCAmelCase,					stride=__lowerCAmelCase,					padding=__lowerCAmelCase						)
							lowercase__					  =				norm_layer(__lowerCAmelCase						) if norm_layer else nn.Identity()




				def 							A__      (						self					:						List[str],					__lowercase					:						Union[str, Any]						):
							lowercase__					  =				self.projection(__lowerCAmelCase						)
							lowercase__					  =				self.norm(__lowerCAmelCase						)
							return embeddings
class 				_snake_case			(					nn.GroupNorm):




				def __init__(						self					:						Tuple,					__lowercase					:						int,					**__lowercase					:						Optional[Any]						):
							super().__init__(1,					__lowerCAmelCase,					**__lowerCAmelCase						)
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						int,					__lowercase					:						Optional[Any]						):
							super().__init__()
							lowercase__					  =				nn.AvgPoolad(__lowerCAmelCase,					stride=1,					padding=pool_size // 2,					count_include_pad=__lowerCAmelCase						)




				def 							A__      (						self					:						Optional[Any],					__lowercase					:						List[Any]						):
							return self.pool(__lowerCAmelCase						) - hidden_states
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Tuple,					__lowercase					:						int,					__lowercase					:						Optional[int],					__lowercase					:						Optional[int],					__lowercase					:						Any						):
							super().__init__()
							lowercase__					  =				nn.Convad(__lowerCAmelCase,					__lowerCAmelCase,					1						)
							lowercase__					  =				nn.Convad(__lowerCAmelCase,					__lowerCAmelCase,					1						)
							lowercase__					  =				PoolFormerDropPath(__lowerCAmelCase						)
							if isinstance(config.hidden_act,					__lowerCAmelCase						):
										lowercase__					  =				ACTaFN[config.hidden_act]
							else:
										lowercase__					  =				config.hidden_act




				def 							A__      (						self					:						Tuple,					__lowercase					:						List[str]						):
							lowercase__					  =				self.conva(__lowerCAmelCase						)
							lowercase__					  =				self.act_fn(__lowerCAmelCase						)
							lowercase__					  =				self.drop(__lowerCAmelCase						)
							lowercase__					  =				self.conva(__lowerCAmelCase						)
							lowercase__					  =				self.drop(__lowerCAmelCase						)

							return hidden_states
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Dict,					__lowercase					:						Optional[Any],					__lowercase					:						int,					__lowercase					:						Dict,					__lowercase					:						Dict,					__lowercase					:						Tuple,					__lowercase					:						Union[str, Any]						):
							super().__init__()
							lowercase__					  =				PoolFormerPooling(__lowerCAmelCase						)
							lowercase__					  =				PoolFormerOutput(__lowerCAmelCase,					__lowerCAmelCase,					__lowerCAmelCase,					__lowerCAmelCase						)
							lowercase__					  =				PoolFormerGroupNorm(__lowerCAmelCase						)
							lowercase__					  =				PoolFormerGroupNorm(__lowerCAmelCase						)

							# Useful for training neural nets
							lowercase__					  =				PoolFormerDropPath(__lowerCAmelCase						) if drop_path > 0.0 else nn.Identity()
							lowercase__					  =				config.use_layer_scale
							if config.use_layer_scale:
										lowercase__					  =				nn.Parameter(
										    config.layer_scale_init_value * torch.ones((__lowerCAmelCase)						),					requires_grad=__lowerCAmelCase						)
										lowercase__					  =				nn.Parameter(
										    config.layer_scale_init_value * torch.ones((__lowerCAmelCase)						),					requires_grad=__lowerCAmelCase						)




				def 							A__      (						self					:						Dict,					__lowercase					:						Dict						):






							if self.use_layer_scale:
										lowercase__					  =				self.pooling(self.before_norm(__lowerCAmelCase						)						)
										lowercase__					  =				self.layer_scale_a.unsqueeze(-1						).unsqueeze(-1						) * pooling_output
										# First residual connection
										lowercase__					  =				hidden_states + self.drop_path(__lowerCAmelCase						)
										lowercase__					  =				()

										lowercase__					  =				self.output(self.after_norm(__lowerCAmelCase						)						)
										lowercase__					  =				self.layer_scale_a.unsqueeze(-1						).unsqueeze(-1						) * layer_output
										# Second residual connection
										lowercase__					  =				hidden_states + self.drop_path(__lowerCAmelCase						)

										lowercase__					  =				(output,) + outputs
										return outputs

							else:
										lowercase__					  =				self.drop_path(self.pooling(self.before_norm(__lowerCAmelCase						)						)						)
										# First residual connection
										lowercase__					  =				pooling_output + hidden_states
										lowercase__					  =				()

										# Second residual connection inside the PoolFormerOutput block
										lowercase__					  =				self.drop_path(self.output(self.after_norm(__lowerCAmelCase						)						)						)
										lowercase__					  =				hidden_states + layer_output

										lowercase__					  =				(output,) + outputs
										return outputs
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Any,					__lowercase					:						Dict						):
							super().__init__()
							lowercase__					  =				config
							# stochastic depth decay rule
							lowercase__					  =				[x.item() for x in torch.linspace(0,					config.drop_path_rate,					sum(config.depths						)						)]

							# patch embeddings
							lowercase__					  =				[]
							for i in range(config.num_encoder_blocks						):
										embeddings.append(
										    PoolFormerEmbeddings(
										        patch_size=config.patch_sizes[i],					stride=config.strides[i],					padding=config.padding[i],					num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1],					hidden_size=config.hidden_sizes[i],					)						)
							lowercase__					  =				nn.ModuleList(__lowerCAmelCase						)

							# Transformer blocks
							lowercase__					  =				[]
							lowercase__					  =				0
							for i in range(config.num_encoder_blocks						):
										# each block consists of layers
										lowercase__					  =				[]
										if i != 0:
													cur += config.depths[i - 1]
										for j in range(config.depths[i]						):
													layers.append(
													    PoolFormerLayer(
													        __lowerCAmelCase,					num_channels=config.hidden_sizes[i],					pool_size=config.pool_size,					hidden_size=config.hidden_sizes[i],					intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio						),					drop_path=dpr[cur + j],					)						)
										blocks.append(nn.ModuleList(__lowerCAmelCase						)						)

							lowercase__					  =				nn.ModuleList(__lowerCAmelCase						)




				def 							A__      (						self					:						Any,					__lowercase					:						Dict,					__lowercase					:						Optional[Any]=False,					__lowercase					:						List[str]=True						):
							lowercase__					  =				() if output_hidden_states else None

							lowercase__					  =				pixel_values
							for idx, layers in enumerate(zip(self.patch_embeddings,					self.block						)						):
										lowercase__  ,       lowercase__					  =				layers
										# Get patch embeddings from hidden_states
										lowercase__					  =				embedding_layer(__lowerCAmelCase						)
										# Send the embeddings through the blocks
										for _, blk in enumerate(__lowerCAmelCase						):
													lowercase__					  =				blk(__lowerCAmelCase						)
													lowercase__					  =				layer_outputs[0]

										if output_hidden_states:
													lowercase__					  =				all_hidden_states + (hidden_states,)

							if not return_dict:
										return tuple(v for v in [hidden_states, all_hidden_states] if v is not None						)

							return BaseModelOutputWithNoAttention(last_hidden_state=__lowerCAmelCase,					hidden_states=__lowerCAmelCase						)
class 				_snake_case			(					UpperCAmelCase__):
				UpperCamelCase__				:					Any     		=PoolFormerConfig
				UpperCamelCase__				:					List[Any]     		="""poolformer"""
				UpperCamelCase__				:					int     		="""pixel_values"""
				UpperCamelCase__				:					Optional[int]     		=True




				def 							A__      (						self					:						List[str],					__lowercase					:						List[str]						):






							if isinstance(__lowerCAmelCase,					(nn.Linear, nn.Convad)						):
										module.weight.data.normal_(mean=0.0,					std=self.config.initializer_range						)
										if module.bias is not None:
													module.bias.data.zero_()
							elif isinstance(__lowerCAmelCase,					nn.LayerNorm						):
										module.bias.data.zero_()
										module.weight.data.fill_(1.0						)




				def 							A__      (						self					:						Optional[int],					__lowercase					:						Optional[int],					__lowercase					:						Optional[Any]=False						):






							if isinstance(__lowerCAmelCase,					__lowerCAmelCase						):
										lowercase__					  =				value
lowercase_								=     r'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

lowercase_								=     r'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`PoolFormerImageProcessor.__call__`] for details.
'''
@add_start_docstrings(
    """The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."""    ,	UpperCAmelCase__    ,	)
class 				_snake_case			(					UpperCAmelCase__):




				def __init__(						self					:						Any,					__lowercase					:						Union[str, Any]						):
							super().__init__(__lowerCAmelCase						)
							lowercase__					  =				config

							lowercase__					  =				PoolFormerEncoder(__lowerCAmelCase						)

							# Initialize weights and apply final processing
							self.post_init()




				def 							A__      (						self					:						List[Any]						):
							return self.embeddings.patch_embeddings




				@add_start_docstrings_to_model_forward(__lowerCAmelCase						)
				@add_code_sample_docstrings(
				    checkpoint=_CHECKPOINT_FOR_DOC,					output_type=__lowerCAmelCase,					config_class=_CONFIG_FOR_DOC,					modality="vision",					expected_output=_EXPECTED_OUTPUT_SHAPE,					)
				def 							A__      (						self					:						int,					__lowercase					:						Optional[torch.FloatTensor] = None,					__lowercase					:						Optional[bool] = None,					__lowercase					:						Optional[bool] = None,					):
							lowercase__					  =				(
							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
							)
							lowercase__					  =				return_dict if return_dict is not None else self.config.use_return_dict

							if pixel_values is None:
										raise ValueError("You have to specify pixel_values"						)

							lowercase__					  =				self.encoder(
							    __lowerCAmelCase,					output_hidden_states=__lowerCAmelCase,					return_dict=__lowerCAmelCase,					)
							lowercase__					  =				encoder_outputs[0]

							if not return_dict:
										return (sequence_output, None) + encoder_outputs[1:]

							return BaseModelOutputWithNoAttention(
							    last_hidden_state=__lowerCAmelCase,					hidden_states=encoder_outputs.hidden_states,					)
class 				_snake_case			(					nn.Module):




				def __init__(						self					:						Union[str, Any],					__lowercase					:						Any						):
							super().__init__()
							lowercase__					  =				nn.Linear(config.hidden_size,					config.hidden_size						)




				def 							A__      (						self					:						int,					__lowercase					:						List[Any]						):
							lowercase__					  =				self.dense(__lowerCAmelCase						)
							return output
@add_start_docstrings(
    """
    PoolFormer Model transformer with an image classification head on top
    """    ,	UpperCAmelCase__    ,	)
class 				_snake_case			(					UpperCAmelCase__):




				def __init__(						self					:						Dict,					__lowercase					:						Optional[int]						):
							super().__init__(__lowerCAmelCase						)
							lowercase__					  =				config.num_labels
							lowercase__					  =				PoolFormerModel(__lowerCAmelCase						)

							# Final norm
							lowercase__					  =				PoolFormerGroupNorm(config.hidden_sizes[-1]						)
							# Classifier head
							lowercase__					  =				(
							    nn.Linear(config.hidden_sizes[-1],					config.num_labels						) if config.num_labels > 0 else nn.Identity()
							)

							# Initialize weights and apply final processing
							self.post_init()




				@add_start_docstrings_to_model_forward(__lowerCAmelCase						)
				@add_code_sample_docstrings(
				    checkpoint=_IMAGE_CLASS_CHECKPOINT,					output_type=__lowerCAmelCase,					config_class=_CONFIG_FOR_DOC,					expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT,					)
				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						Optional[torch.FloatTensor] = None,					__lowercase					:						Optional[torch.LongTensor] = None,					__lowercase					:						Optional[bool] = None,					__lowercase					:						Optional[bool] = None,					):
							lowercase__					  =				return_dict if return_dict is not None else self.config.use_return_dict

							lowercase__					  =				self.poolformer(
							    __lowerCAmelCase,					output_hidden_states=__lowerCAmelCase,					return_dict=__lowerCAmelCase,					)

							lowercase__					  =				outputs[0]

							lowercase__					  =				self.classifier(self.norm(__lowerCAmelCase						).mean([-2, -1]						)						)

							lowercase__					  =				None
							if labels is not None:
										if self.config.problem_type is None:
													if self.num_labels == 1:
																lowercase__					  =				"regression"
													elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
																lowercase__					  =				"single_label_classification"
													else:
																lowercase__					  =				"multi_label_classification"

										if self.config.problem_type == "regression":
													lowercase__					  =				MSELoss()
													if self.num_labels == 1:
																lowercase__					  =				loss_fct(logits.squeeze(),					labels.squeeze()						)
													else:
																lowercase__					  =				loss_fct(__lowerCAmelCase,					__lowerCAmelCase						)
										elif self.config.problem_type == "single_label_classification":
													lowercase__					  =				CrossEntropyLoss()
													lowercase__					  =				loss_fct(logits.view(-1,					self.num_labels						),					labels.view(-1						)						)
										elif self.config.problem_type == "multi_label_classification":
													lowercase__					  =				BCEWithLogitsLoss()
													lowercase__					  =				loss_fct(__lowerCAmelCase,					__lowerCAmelCase						)

							if not return_dict:
										lowercase__					  =				(logits,) + outputs[2:]
										return ((loss,) + output) if loss is not None else output

							return ImageClassifierOutputWithNoAttention(loss=__lowerCAmelCase,					logits=__lowerCAmelCase,					hidden_states=outputs.hidden_states						)







# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				{
			    "en": "Machine learning is great, isn't it?",
			    "ru": "Машинное обучение - это здорово, не так ли?",
			    "de": "Maschinelles Lernen ist großartig, oder?",
			}

			# BLUE scores as follows:
			# "pair": [fairseq, transformers]
			lowercase__					  =				{
			    "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "39.20"],
			    "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.47"],
			    "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.83"],
			    "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.35"],
			}
			lowercase__					  =				f'''{src_lang}-{tgt_lang}'''

			lowercase__					  =				f'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
			os.makedirs(SCREAMING_SNAKE_CASE_						,  exist_ok=SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				os.path.join(SCREAMING_SNAKE_CASE_						,  "README.md"  )
			print(f'''Generating {path}'''  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"						,  encoding="utf-8"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )


# make sure we are under the root of the project
lowercase_								=     Path(__file__).resolve().parent.parent.parent
lowercase_								=     repo_dir / """model_cards"""

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
							lowercase_				,    lowercase_				,    lowercase_								=     model_name.split("""-""")
							lowercase_								=     model_cards_dir / """facebook""" / model_name
							write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)




