





import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length





def lowerCAmelCase__(__snake_case							)       ->   Any:







	'''simple docstring'''







	lowerCamelCase__              =	int(__snake_case							)
	lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	t // 3600, (t // 60) % 60, t % 60
	return F'{h}:{m:02d}:{s:02d}' if h != 0 else F'{m:02d}:{s:02d}'





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case=300							)       ->   Tuple:







	'''simple docstring'''







	return F'\n    <div>\n      {prefix}\n      <progress value=\'{value}\' max=\'{total}\' style=\'width:{width}px; height:20px; vertical-align: middle;\'></progress>\n      {label}\n    </div>\n    '





def lowerCAmelCase__(__snake_case							)       ->   Optional[int]:







	'''simple docstring'''







	lowerCamelCase__              =	'''<table border="1" class="dataframe">\n'''
	html_code += """  <thead>\n <tr style="text-align: left;">\n"""
	for i in items[0]:
		html_code += F'      <th>{i}</th>\n'
	html_code += "    </tr>\n  </thead>\n  <tbody>\n"
	for line in items[1:]:
		html_code += "    <tr>\n"
		for elt in line:
			lowerCamelCase__              =	F'{elt:.6f}' if isinstance(__snake_case ,__snake_case							) else str(__snake_case							)
			html_code += F'      <td>{elt}</td>\n'
		html_code += "    </tr>\n"
	html_code += "  </tbody>\n</table><p>"
	return html_code






class       __A					:

			'''simple docstring'''







			lowerCAmelCase_							  =		5
			lowerCAmelCase_							  =		0.2





			def __init__(					self ,						__lowerCAmelCase ,						__lowerCAmelCase = None ,						__lowerCAmelCase = True ,						__lowerCAmelCase = None ,						__lowerCAmelCase = 3_0_0 ,						):



				'''simple docstring'''







				lowerCamelCase__              =	total
				lowerCamelCase__              =	'''''' if prefix is None else prefix
				lowerCamelCase__              =	leave
				lowerCamelCase__              =	parent
				lowerCamelCase__              =	width
				lowerCamelCase__              =	None
				lowerCamelCase__              =	None
				lowerCamelCase__              =	None





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase = False ,						__lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__              =	value
				if comment is not None:
					lowerCamelCase__              =	comment
				if self.last_value is None:
					lowerCamelCase__              =	lowerCamelCase__              =	time.time()
					lowerCamelCase__              =	lowerCamelCase__              =	value
					lowerCamelCase__              =	lowerCamelCase__              =	None
					lowerCamelCase__              =	self.warmup
					lowerCamelCase__              =	1
					self.update_bar(__lowerCAmelCase							)
				elif value <= self.last_value and not force_update:
					return
				elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for ,						self.total							):
					if self.first_calls > 0:
						self.first_calls -= 1
					lowerCamelCase__              =	time.time()
					lowerCamelCase__              =	current_time - self.start_time
					# We could have value = self.start_value if the update is called twixe with the same start value.
					if value > self.start_value:
						lowerCamelCase__              =	self.elapsed_time / (value - self.start_value)
					else:
						lowerCamelCase__              =	None
					if value >= self.total:
						lowerCamelCase__              =	self.total
						lowerCamelCase__              =	None
						if not self.leave:
							self.close()
					elif self.average_time_per_item is not None:
						lowerCamelCase__              =	self.average_time_per_item * (self.total - value)
					self.update_bar(__lowerCAmelCase							)
					lowerCamelCase__              =	value
					lowerCamelCase__              =	current_time
					if self.average_time_per_item is None:
						lowerCamelCase__              =	1
					else:
						lowerCamelCase__              =	max(int(self.update_every / self.average_time_per_item							) ,						1							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None							):



				'''simple docstring'''







				lowerCamelCase__              =	''' ''' * (len(str(self.total							)							) - len(str(__lowerCAmelCase							)							)) + str(__lowerCAmelCase							)
				if self.elapsed_time is None:
					lowerCamelCase__              =	F'[{spaced_value}/{self.total} : < :'
				elif self.predicted_remaining is None:
					lowerCamelCase__              =	F'[{spaced_value}/{self.total} {format_time(self.elapsed_time							)}'
				else:
					lowerCamelCase__              =	(
					    F'[{spaced_value}/{self.total} {format_time(self.elapsed_time							)} <'
					    F' {format_time(self.predicted_remaining							)}'
					)
					self.label += F', {1/self.average_time_per_item:.2f} it/s'
				self.label += "]" if self.comment is None or len(self.comment							) == 0 else F', {self.comment}]'
				self.display()





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	html_progress_bar(self.value ,						self.total ,						self.prefix ,						self.label ,						self.width							)
				if self.parent is not None:
					# If this is a child bar, the parent will take care of the display.
					self.parent.display()
					return
				if self.output is None:
					lowerCamelCase__              =	disp.display(disp.HTML(self.html_code							) ,						display_id=__lowerCAmelCase							)
				else:
					self.output.update(disp.HTML(self.html_code							)							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				if self.parent is None and self.output is not None:
					self.output.update(disp.HTML(''''''							)							)






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			def __init__(					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None							):



				'''simple docstring'''







				super().__init__(__lowerCAmelCase							)
				lowerCamelCase__              =	None if column_names is None else [column_names]
				lowerCamelCase__              =	None





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	html_progress_bar(self.value ,						self.total ,						self.prefix ,						self.label ,						self.width							)
				if self.inner_table is not None:
					self.html_code += text_to_html_table(self.inner_table							)
				if self.child_bar is not None:
					self.html_code += self.child_bar.html_code
				if self.output is None:
					lowerCamelCase__              =	disp.display(disp.HTML(self.html_code							) ,						display_id=__lowerCAmelCase							)
				else:
					self.output.update(disp.HTML(self.html_code							)							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if self.inner_table is None:
					lowerCamelCase__              =	[list(values.keys()							), list(values.values()							)]
				else:
					lowerCamelCase__              =	self.inner_table[0]
					if len(self.inner_table							) == 1:
						# We give a chance to update the column names at the first iteration
						for key in values.keys():
							if key not in columns:
								columns.append(__lowerCAmelCase							)
						lowerCamelCase__              =	columns
					self.inner_table.append([values[c] for c in columns]							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						__lowerCAmelCase=3_0_0							):



				'''simple docstring'''







				lowerCamelCase__              =	NotebookProgressBar(__lowerCAmelCase ,						prefix=__lowerCAmelCase ,						parent=self ,						width=__lowerCAmelCase							)
				return self.child_bar





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	None
				self.display()






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			def __init__(					self							):



				'''simple docstring'''







				lowerCamelCase__              =	None
				lowerCamelCase__              =	None
				lowerCamelCase__              =	False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	'''Epoch''' if args.evaluation_strategy == IntervalStrategy.EPOCH else '''Step'''
				lowerCamelCase__              =	0
				lowerCamelCase__              =	0
				lowerCamelCase__              =	[self.first_column] + ['''Training Loss''']
				if args.evaluation_strategy != IntervalStrategy.NO:
					column_names.append('''Validation Loss'''							)
				lowerCamelCase__              =	NotebookTrainingTracker(state.max_steps ,						__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	int(state.epoch							) if int(state.epoch							) == state.epoch else F'{state.epoch:.2f}'
				self.training_tracker.update(
				    state.global_step + 1 ,						comment=F'Epoch {epoch}/{state.num_train_epochs}' ,						force_update=self._force_next_update ,						)
				lowerCamelCase__              =	False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						**__lowerCAmelCase							):



				'''simple docstring'''







				if not has_length(__lowerCAmelCase							):
					return
				if self.prediction_bar is None:
					if self.training_tracker is not None:
						lowerCamelCase__              =	self.training_tracker.add_child(len(__lowerCAmelCase							)							)
					else:
						lowerCamelCase__              =	NotebookProgressBar(len(__lowerCAmelCase							)							)
					self.prediction_bar.update(1							)
				else:
					self.prediction_bar.update(self.prediction_bar.value + 1							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				if self.prediction_bar is not None:
					self.prediction_bar.close()
				lowerCamelCase__              =	None





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						**__lowerCAmelCase							):



				'''simple docstring'''







				if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
					lowerCamelCase__              =	{'''Training Loss''': logs['''loss''']}
					# First column is necessarily Step sine we're not in epoch eval strategy
					lowerCamelCase__              =	state.global_step
					self.training_tracker.write_line(__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						**__lowerCAmelCase							):



				'''simple docstring'''







				if self.training_tracker is not None:
					lowerCamelCase__              =	{'''Training Loss''': '''No log''', '''Validation Loss''': '''No log'''}
					for log in reversed(state.log_history							):
						if "loss" in log:
							lowerCamelCase__              =	log['''loss''']
							break

					if self.first_column == "Epoch":
						lowerCamelCase__              =	int(state.epoch							)
					else:
						lowerCamelCase__              =	state.global_step
					lowerCamelCase__              =	'''eval'''
					for k in metrics:
						if k.endswith('''_loss'''							):
							lowerCamelCase__              =	re.sub(r'''\_loss$''' ,						'''''' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop('''total_flos''' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop('''epoch''' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop(F'{metric_key_prefix}_runtime' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop(F'{metric_key_prefix}_samples_per_second' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop(F'{metric_key_prefix}_steps_per_second' ,						__lowerCAmelCase							)
					lowerCamelCase__              =	metrics.pop(F'{metric_key_prefix}_jit_compilation_time' ,						__lowerCAmelCase							)
					for k, v in metrics.items():
						if k == F'{metric_key_prefix}_loss':
							lowerCamelCase__              =	v
						else:
							lowerCamelCase__              =	k.split('''_'''							)
							lowerCamelCase__              =	''' '''.join([part.capitalize() for part in splits[1:]]							)
							lowerCamelCase__              =	v
					self.training_tracker.write_line(__lowerCAmelCase							)
					self.training_tracker.remove_child()
					lowerCamelCase__              =	None
					# Evaluation takes a long time so we should force the next update.
					lowerCamelCase__              =	True





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				self.training_tracker.update(
				    state.global_step ,						comment=F'Epoch {int(state.epoch							)}/{state.num_train_epochs}' ,						force_update=__lowerCAmelCase							)
				lowerCamelCase__              =	None










from dataclasses import dataclass
from typing import Optional

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .modeling_utils import ModelMixin






@dataclass
class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		42






class       __A					(					lowerCAmelCase    ,       lowerCAmelCase				):

			'''simple docstring'''







			@register_to_config
			def __init__(					self ,						__lowerCAmelCase = 1_6 ,						__lowerCAmelCase = 8_8 ,						__lowerCAmelCase = None ,						__lowerCAmelCase = None ,						__lowerCAmelCase = 1 ,						__lowerCAmelCase = 0.0 ,						__lowerCAmelCase = 3_2 ,						__lowerCAmelCase = None ,						__lowerCAmelCase = False ,						__lowerCAmelCase = None ,						__lowerCAmelCase = "geglu" ,						__lowerCAmelCase = True ,						__lowerCAmelCase = True ,						):



				'''simple docstring'''







				super().__init__()
				lowerCamelCase__              =	num_attention_heads
				lowerCamelCase__              =	attention_head_dim
				lowerCamelCase__              =	num_attention_heads * attention_head_dim

				lowerCamelCase__              =	in_channels

				lowerCamelCase__              =	torch.nn.GroupNorm(num_groups=__lowerCAmelCase ,						num_channels=__lowerCAmelCase ,						eps=1E-6 ,						affine=__lowerCAmelCase							)
				lowerCamelCase__              =	nn.Linear(__lowerCAmelCase ,						__lowerCAmelCase							)

				# 3. Define transformers blocks
				lowerCamelCase__              =	nn.ModuleList(
				    [
				        BasicTransformerBlock(
				            __lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						dropout=__lowerCAmelCase ,						cross_attention_dim=__lowerCAmelCase ,						activation_fn=__lowerCAmelCase ,						attention_bias=__lowerCAmelCase ,						double_self_attention=__lowerCAmelCase ,						norm_elementwise_affine=__lowerCAmelCase ,						)
				        for d in range(__lowerCAmelCase							)
				    ]							)

				lowerCamelCase__              =	nn.Linear(__lowerCAmelCase ,						__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						__lowerCAmelCase=None ,						__lowerCAmelCase=1 ,						__lowerCAmelCase=None ,						__lowerCAmelCase = True ,						):



				'''simple docstring'''







				lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	hidden_states.shape
				lowerCamelCase__              =	batch_frames // num_frames

				lowerCamelCase__              =	hidden_states

				lowerCamelCase__              =	hidden_states[None, :].reshape(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
				lowerCamelCase__              =	hidden_states.permute(0 ,						2 ,						1 ,						3 ,						4							)

				lowerCamelCase__              =	self.norm(__lowerCAmelCase							)
				lowerCamelCase__              =	hidden_states.permute(0 ,						3 ,						4 ,						2 ,						1							).reshape(batch_size * height * width ,						__lowerCAmelCase ,						__lowerCAmelCase							)

				lowerCamelCase__              =	self.proj_in(__lowerCAmelCase							)

				# 2. Blocks
				for block in self.transformer_blocks:
					lowerCamelCase__              =	block(
					    __lowerCAmelCase ,						encoder_hidden_states=__lowerCAmelCase ,						timestep=__lowerCAmelCase ,						cross_attention_kwargs=__lowerCAmelCase ,						class_labels=__lowerCAmelCase ,						)

				# 3. Output
				lowerCamelCase__              =	self.proj_out(__lowerCAmelCase							)
				lowerCamelCase__              =	(
				    hidden_states[None, None, :]
				    .reshape(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
				    .permute(0 ,						3 ,						4 ,						1 ,						2							)
				    .contiguous()
				)
				lowerCamelCase__              =	hidden_states.reshape(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)

				lowerCamelCase__              =	hidden_states + residual

				if not return_dict:
					return (output,)

				return TransformerTemporalModelOutput(sample=__lowerCAmelCase							)



