


'''simple docstring'''



import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class  UpperCamelCase__(     lowerCAmelCase		,       unittest.TestCase ):
				__magic_name__       :   int							= CTRLTokenizer
				__magic_name__       :   int							= False
				__magic_name__       :   Optional[Any]							= False




				def   a__(	self    :				int   )->  Optional[Any]:

										"""simple docstring"""




										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										UpperCAmelCase         =			["""adapt""", """re@@""", """a@@""", """apt""", """c@@""", """t""", """<unk>"""]
										UpperCAmelCase         =			dict(zip(A__   ,					range(len(A__   )   )   )   )
										UpperCAmelCase         =			["""#version: 0.2""", """a p""", """ap t</w>""", """r e""", """a d""", """ad apt</w>""", """"""]
										UpperCAmelCase         =			{"""unk_token""": """<unk>"""}

										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''vocab_file''']   )
										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''merges_file''']   )
										with open(self.vocab_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as fp:
																fp.write(json.dumps(A__   ) + '''\n'''   )
										with open(self.merges_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as fp:
																fp.write('''\n'''.join(A__   )   )




				def   a__(	self    :				Any   ,					**lowerCAmelCase    :				Any   )->  List[str]:

										"""simple docstring"""




										kwargs.update(self.special_tokens_map   )
										return CTRLTokenizer.from_pretrained(self.tmpdirname   ,					**A__   )




				def   a__(	self    :				Any   ,					lowerCAmelCase    :				List[str]   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			"""adapt react readapt apt"""
										UpperCAmelCase         =			"""adapt react readapt apt"""
										return input_text, output_text





				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			CTRLTokenizer(self.vocab_file   ,					self.merges_file   ,					**self.special_tokens_map   )
										UpperCAmelCase         =			"""adapt react readapt apt"""
										UpperCAmelCase         =			"""adapt re@@ a@@ c@@ t re@@ adapt apt""".split()
										UpperCAmelCase         =			tokenizer.tokenize(A__   )
										self.assertListEqual(A__   ,					A__   )

										UpperCAmelCase         =			tokens + [tokenizer.unk_token]

										UpperCAmelCase         =			[0, 1, 2, 4, 5, 1, 0, 3, 6]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(A__   )   ,					A__   )









'''simple docstring'''



import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
	import torch

	from transformers.generation import DisjunctiveConstraint






@require_torch
class  UpperCamelCase__(     unittest.TestCase ):




				def   a__(	self    :				Optional[int]   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 4], [1, 2, 3, 4]]
										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )
										self.assertTrue(isinstance(dc.token_ids   ,					lowerCAmelCase   )   )

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]   )   )

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint([torch.LongTensor([1, 2, 4]   ), torch.LongTensor([1, 2, 3, 4, 5]   )]   )




				def   a__(	self    :				Union[str, Any]   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2], [1, 2, 3, 4]]

										with self.assertRaises(lowerCAmelCase   ):
																DisjunctiveConstraint(lowerCAmelCase   )  # fails here




				def   a__(	self    :				Any   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 3], [1, 2, 4]]

										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										UpperCAmelCase         =			stepped is True and completed is False and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										UpperCAmelCase         =			stepped is True and completed is False and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(3   )
										UpperCAmelCase         =			stepped is True and completed is True and reset is False
										self.assertTrue(lowerCAmelCase   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.current_seq == [1, 2, 3]   )





				def   a__(	self    :				int   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			[[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

										UpperCAmelCase         =			DisjunctiveConstraint(lowerCAmelCase   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(4   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.current_seq == [1, 2, 4]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(5   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.current_seq == [1, 2, 4, 5]   )

										dc.reset()

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(1   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.remaining() == 3   )
										self.assertTrue(dc.current_seq == [1]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(2   )
										self.assertTrue(not dc.completed   )
										self.assertTrue(dc.remaining() == 2   )
										self.assertTrue(dc.current_seq == [1, 2]   )

										UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			dc.update(5   )
										self.assertTrue(dc.completed   )  # Completed!
										self.assertTrue(dc.remaining() == 0   )
										self.assertTrue(dc.current_seq == [1, 2, 5]   )





