





from random import shuffle

import tensorflow as tf
from numpy import array


def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict					,   UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      int(UpperCamelCase__    )
		assert noofclusters < len(UpperCamelCase__    )

		# Find out the dimensionality
		SCREAMING_SNAKE_CASE__			=      len(vectors[0]    )

		# Will help select random centroids from among the available vectors
		SCREAMING_SNAKE_CASE__			=      list(range(len(UpperCamelCase__    )    )    )
		shuffle(UpperCamelCase__    )

		# GRAPH OF COMPUTATION
		# We initialize a new graph and set it as the default during each run
		# of this algorithm. This ensures that as this function is called
		# multiple times, the default graph doesn't keep getting crowded with
		# unused ops and Variables from previous function calls.

		SCREAMING_SNAKE_CASE__			=      tf.Graph()

		with graph.as_default():
				# SESSION OF COMPUTATION

				SCREAMING_SNAKE_CASE__			=      tf.Session()

				##CONSTRUCTING THE ELEMENTS OF COMPUTATION

				##First lets ensure we have a Variable vector for each centroid,
				##initialized to one of the vectors from the available data points
				SCREAMING_SNAKE_CASE__			=      [
				    tf.Variable(vectors[vector_indices[i]]    ) for i in range(UpperCamelCase__    )
				]
				##These nodes will assign the centroid Variables the appropriate
				##values
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""float64"""					,   [dim]    )
				SCREAMING_SNAKE_CASE__			=      []
				for centroid in centroids:
						cent_assigns.append(tf.assign(UpperCamelCase__					,   UpperCamelCase__    )    )

				##Variables for cluster assignments of individual vectors(initialized
				##to 0 at first)
				SCREAMING_SNAKE_CASE__			=      [tf.Variable(0    ) for i in range(len(UpperCamelCase__    )    )]
				##These nodes will assign an assignment Variable the appropriate
				##value
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""int32"""    )
				SCREAMING_SNAKE_CASE__			=      []
				for assignment in assignments:
						cluster_assigns.append(tf.assign(UpperCamelCase__					,   UpperCamelCase__    )    )

				##Now lets construct the node that will compute the mean
				# The placeholder for the input
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""float"""					,   [None, dim]    )
				# The Node/op takes the input and computes a mean along the 0th
				# dimension, i.e. the list of input vectors
				SCREAMING_SNAKE_CASE__			=      tf.reduce_mean(UpperCamelCase__					,   0    )

				##Node for computing Euclidean distances
				# Placeholders for input
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""float"""					,   [dim]    )
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""float"""					,   [dim]    )
				SCREAMING_SNAKE_CASE__			=      tf.sqrt(tf.reduce_sum(tf.pow(tf.sub(UpperCamelCase__					,   UpperCamelCase__    )					,   2    )    )    )

				##This node will figure out which cluster to assign a vector to,
				##based on Euclidean distances of the vector from the centroids.
				# Placeholder for input
				SCREAMING_SNAKE_CASE__			=      tf.placeholder("""float"""					,   [noofclusters]    )
				SCREAMING_SNAKE_CASE__			=      tf.argmin(UpperCamelCase__					,   0    )

				##INITIALIZING STATE VARIABLES

				##This will help initialization of all Variables defined with respect
				##to the graph. The Variable-initializer should be defined after
				##all the Variables have been constructed, so that each of them
				##will be included in the initialization.
				SCREAMING_SNAKE_CASE__			=      tf.initialize_all_variables()

				# Initialize all variables
				sess.run(UpperCamelCase__    )

				##CLUSTERING ITERATIONS

				# Now perform the Expectation-Maximization steps of K-Means clustering
				# iterations. To keep things simple, we will only do a set number of
				# iterations, instead of using a Stopping Criterion.
				SCREAMING_SNAKE_CASE__			=      100
				for _ in range(UpperCamelCase__    ):
						##EXPECTATION STEP
						##Based on the centroid locations till last iteration, compute
						##the _expected_ centroid assignments.
						# Iterate over each vector
						for vector_n in range(len(UpperCamelCase__    )    ):
								SCREAMING_SNAKE_CASE__			=      vectors[vector_n]
								# Compute Euclidean distance between this vector and each
								# centroid. Remember that this list cannot be named
								#'centroid_distances', since that is the input to the
								# cluster assignment node.
								SCREAMING_SNAKE_CASE__			=      [
								    sess.run(UpperCamelCase__					,   feed_dict={va: vect, va: sess.run(UpperCamelCase__    )}    )
								    for centroid in centroids
								]
								# Now use the cluster assignment node, with the distances
								# as the input
								SCREAMING_SNAKE_CASE__			=      sess.run(
								    UpperCamelCase__					,   feed_dict={centroid_distances: distances}    )
								# Now assign the value to the appropriate state variable
								sess.run(
								    cluster_assigns[vector_n]					,   feed_dict={assignment_value: assignment}    )

						##MAXIMIZATION STEP
						# Based on the expected state computed from the Expectation Step,
						# compute the locations of the centroids so as to maximize the
						# overall objective of minimizing within-cluster Sum-of-Squares
						for cluster_n in range(UpperCamelCase__    ):
								# Collect all the vectors assigned to this cluster
								SCREAMING_SNAKE_CASE__			=      [
								    vectors[i]
								    for i in range(len(UpperCamelCase__    )    )
								    if sess.run(assignments[i]    ) == cluster_n
								]
								# Compute new centroid location
								SCREAMING_SNAKE_CASE__			=      sess.run(
								    UpperCamelCase__					,   feed_dict={mean_input: array(UpperCamelCase__    )}    )
								# Assign value to appropriate variable
								sess.run(
								    cent_assigns[cluster_n]					,   feed_dict={centroid_value: new_location}    )

        # Return centroids and assignments
				SCREAMING_SNAKE_CASE__			=      sess.run(UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      sess.run(UpperCamelCase__    )
				return centroids, assignments






import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_lowerCamelCase   				=			data_utils.TransfoXLTokenizer
_lowerCamelCase   				=			data_utils.TransfoXLCorpus
_lowerCamelCase   				=			data_utils
_lowerCamelCase   				=			data_utils






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Dict					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Tuple    ):
		if transfo_xl_dataset_file:
				# Convert a pre-processed corpus (see original TensorFlow repo)
				with open(UpperCamelCase__					,   """rb"""    ) as fp:
						SCREAMING_SNAKE_CASE__			=      pickle.load(UpperCamelCase__					,   encoding="""latin1"""    )
				# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""pretrained_vocab_file"""]
				print(f'''Save vocabulary to {pytorch_vocab_dump_path}'''    )
				SCREAMING_SNAKE_CASE__			=      corpus.vocab.__dict__
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      corpus.__dict__
				corpus_dict_no_vocab.pop("""vocab"""					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      pytorch_dump_folder_path + """/""" + CORPUS_NAME
				print(f'''Save dataset to {pytorch_dataset_dump_path}'''    )
				torch.save(UpperCamelCase__					,   UpperCamelCase__    )

		if tf_checkpoint_path:
				# Convert a pre-trained TensorFlow model
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.abspath(UpperCamelCase__    )

				print(f'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''    )
				# Initialise PyTorch model
				if transfo_xl_config_file == "":
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig()
				else:
						SCREAMING_SNAKE_CASE__			=      TransfoXLConfig.from_json_file(UpperCamelCase__    )
				print(f'''Building PyTorch model from configuration: {config}'''    )
				SCREAMING_SNAKE_CASE__			=      TransfoXLLMHeadModel(UpperCamelCase__    )

				SCREAMING_SNAKE_CASE__			=      load_tf_weights_in_transfo_xl(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
				# Save pytorch-model
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   UpperCamelCase__    )
				print(f'''Save PyTorch model to {os.path.abspath(UpperCamelCase__    )}'''    )
				torch.save(model.state_dict()					,   UpperCamelCase__    )
				print(f'''Save configuration file to {os.path.abspath(UpperCamelCase__    )}'''    )
				with open(UpperCamelCase__					,   """w"""					,   encoding="""utf-8"""    ) as f:
						f.write(config.to_json_string()    )


if __name__ == "__main__":
	_lowerCamelCase   				=			argparse.ArgumentParser()
	parser.add_argument(
	    '--pytorch_dump_folder_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the folder to store the PyTorch model or dataset/vocab.',
	)
	parser.add_argument(
	    '--tf_checkpoint_path',
	    default='',
	    type=str,
	    help='An optional path to a TensorFlow checkpoint path to be converted.',
	)
	parser.add_argument(
	    '--transfo_xl_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained BERT model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--transfo_xl_dataset_file',
	    default='',
	    type=str,
	    help='An optional dataset file to be converted in a vocabulary.',
	)
	_lowerCamelCase   				=			parser.parse_args()
	convert_transfo_xl_checkpoint_to_pytorch(
	    args.tf_checkpoint_path,
	    args.transfo_xl_config_file,
	    args.pytorch_dump_folder_path,
	    args.transfo_xl_dataset_file,
	)