





import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


_a						=   {
    "cola": 2,
    "mnli": 3,
    "mrpc": 2,
    "sst-2": 2,
    "sts-b": 1,
    "qqp": 2,
    "qnli": 2,
    "rte": 2,
    "wnli": 2,
}


logging.set_verbosity_info()





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case=None							)       ->   int:







	'''simple docstring'''







	lowerCamelCase__              =	XLNetConfig.from_json_file(__snake_case							)

	lowerCamelCase__              =	finetuning_task.lower() if finetuning_task is not None else ''''''
	if finetuning_task in GLUE_TASKS_NUM_LABELS:
		print(F'Building PyTorch XLNetForSequenceClassification model from configuration: {config}'							)
		lowerCamelCase__              =	finetuning_task
		lowerCamelCase__              =	GLUE_TASKS_NUM_LABELS[finetuning_task]
		lowerCamelCase__              =	XLNetForSequenceClassification(__snake_case							)
	elif "squad" in finetuning_task:
		lowerCamelCase__              =	finetuning_task
		lowerCamelCase__              =	XLNetForQuestionAnswering(__snake_case							)
	else:
		lowerCamelCase__              =	XLNetLMHeadModel(__snake_case							)

	# Load weights from tf checkpoint
	load_tf_weights_in_xlnet(__snake_case ,__snake_case ,__snake_case							)

	# Save pytorch-model
	lowerCamelCase__              =	os.path.join(__snake_case ,__snake_case							)
	lowerCamelCase__              =	os.path.join(__snake_case ,__snake_case							)
	print(F'Save PyTorch model to {os.path.abspath(__snake_case							)}'							)
	torch.save(model.state_dict() ,__snake_case							)
	print(F'Save configuration file to {os.path.abspath(__snake_case							)}'							)
	with open(__snake_case ,'''w''' ,encoding='''utf-8'''							) as f:
		f.write(config.to_json_string()							)


if __name__ == "__main__":
			_a						=   argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
			)
			parser.add_argument(
			    "--xlnet_config_file",
			    default=None,
			    type=str,
			    required=True,
			    help=(
			        "The config json file corresponding to the pre-trained XLNet model. \n"
			        "This specifies the model architecture."
			    ),
			)
			parser.add_argument(
			    "--pytorch_dump_folder_path",
			    default=None,
			    type=str,
			    required=True,
			    help="Path to the folder to store the PyTorch model or dataset/vocab.",
			)
			parser.add_argument(
			    "--finetuning_task",
			    default=None,
			    type=str,
			    help="Name of a task on which the XLNet TensorFlow model was fine-tuned",
			)
			_a						=   parser.parse_args()
			print(args)

			convert_xlnet_checkpoint_to_pytorch(
			    args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
			)










import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


_a						=   logging.get_logger(__name__)






class       __A					:

			'''simple docstring'''







			lowerCAmelCase_							  =		None





@experimental
def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case							)       ->   Tuple:







	'''simple docstring'''







	if ParallelBackendConfig.backend_name is None:
		return _map_with_multiprocessing_pool(
		    __snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case							)

	return _map_with_joblib(__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case							)       ->   int:







	'''simple docstring'''







	lowerCamelCase__              =	num_proc if num_proc <= len(__snake_case							) else len(__snake_case							)
	lowerCamelCase__              =	[]  # We organize the splits ourselve (contiguous splits)
	for index in range(__snake_case							):
		lowerCamelCase__              =	len(__snake_case							) // num_proc
		lowerCamelCase__              =	len(__snake_case							) % num_proc
		lowerCamelCase__              =	div * index + min(__snake_case ,__snake_case							)
		lowerCamelCase__              =	start + div + (1 if index < mod else 0)
		split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)							)

	if len(__snake_case							) != sum(len(i[1]							) for i in split_kwds							):
		raise ValueError(
		    F'Error dividing inputs iterable among processes. '
		    F'Total number of objects {len(__snake_case							)}, '
		    F'length: {sum(len(i[1]							) for i in split_kwds							)}'							)

	logger.info(
	    F'Spawning {num_proc} processes for {len(__snake_case							)} objects in slices of {[len(i[1]							) for i in split_kwds]}'							)
	lowerCamelCase__       ,			lowerCamelCase__              =	None, None
	if not disable_tqdm:
		lowerCamelCase__       ,			lowerCamelCase__              =	(RLock(),), tqdm.set_lock
	with Pool(__snake_case ,initargs=__snake_case ,initializer=__snake_case							) as pool:
		lowerCamelCase__              =	pool.map(__snake_case ,__snake_case							)
	logger.info(F'Finished {num_proc} processes'							)
	lowerCamelCase__              =	[obj for proc_res in mapped for obj in proc_res]
	logger.info(F'Unpacked {len(__snake_case							)} objects'							)

	return mapped





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case ,__snake_case							)       ->   List[str]:







	'''simple docstring'''







	import joblib

	with joblib.parallel_backend(ParallelBackendConfig.backend_name ,n_jobs=__snake_case							):
		return joblib.Parallel()(
		    joblib.delayed(__snake_case							)((function, obj, types, None, True, None)							) for obj in iterable							)





@experimental
@contextlib.contextmanager
def lowerCAmelCase__(__snake_case							)       ->   int:







	'''simple docstring'''







	lowerCamelCase__              =	backend_name

	if backend_name == "spark":
		from joblibspark import register_spark

		register_spark()

		# TODO: call create_cache_and_write_probe if "download" in steps
		# TODO: raise NotImplementedError when Dataset.map etc is called

	try:
		yield
	finally:
		lowerCamelCase__              =	None



