






"""simple docstring"""



import logging
import os
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import List, Optional

import faiss
import torch
from datasets import Features, Sequence, Value, load_dataset

from transformers import DPRContextEncoder, DPRContextEncoderTokenizerFast, HfArgumentParser


A_    =							logging.getLogger(__name__)
torch.set_grad_enabled(False)
A_    =							'''cuda''' if torch.cuda.is_available() else '''cpu'''




def      _lowerCAmelCase		(							UpperCAmelCase__       :					str,						UpperCAmelCase__       :					Optional[int]=1_0_0,						UpperCAmelCase__       :					Tuple=" "      )		->List[str]:
						A__						:  Optional[Any]		  =  text.split(UpperCAmelCase__      )
						return [character.join(text[i : i + n]      ).strip() for i in range(0,						len(UpperCAmelCase__      ),						UpperCAmelCase__      )]




def      _lowerCAmelCase		(							UpperCAmelCase__       :					dict      )		->dict:
						A__  ,  A__						:  List[str]		  =  [], []
						for title, text in zip(documents["""title"""],						documents["""text"""]      ):
												if text is not None:
																		for passage in split_text(UpperCAmelCase__      ):
																								titles.append(title if title is not None else """"""      )
																								texts.append(UpperCAmelCase__      )
						return {"title": titles, "text": texts}




def      _lowerCAmelCase		(							UpperCAmelCase__       :					dict,						UpperCAmelCase__       :					DPRContextEncoder,						UpperCAmelCase__       :					DPRContextEncoderTokenizerFast      )		->dict:
						A__						:  Any		  =  ctx_tokenizer(
						    documents["""title"""],						documents["""text"""],						truncation=UpperCAmelCase__,						padding="""longest""",						return_tensors="""pt"""      )["""input_ids"""]
						A__						:  str		  =  ctx_encoder(input_ids.to(device=UpperCAmelCase__      ),						return_dict=UpperCAmelCase__      ).pooler_output
						return {"embeddings": embeddings.detach().cpu().numpy()}




def      _lowerCAmelCase		(							UpperCAmelCase__       :					"RagExampleArguments",						UpperCAmelCase__       :					"ProcessingArguments",						UpperCAmelCase__       :					"IndexHnswArguments",						)		->Any:
						######################################
						logger.info("""Step 1 - Create the dataset"""      )
						######################################

						# The dataset needed for RAG must have three columns:
						# - title (string): title of the document
						# - text (string): text of a passage of the document
						# - embeddings (array of dimension d): DPR representation of the passage
						# Let's say you have documents in tab-separated csv files with columns "title" and "text"
						assert os.path.isfile(rag_example_args.csv_path      ), "Please provide a valid path to a csv file"

						# You can load a Dataset object this way
						A__						:  Optional[Any]		  =  load_dataset(
						    """csv""",						data_files=[rag_example_args.csv_path],						split="""train""",						delimiter="""\t""",						column_names=["""title""", """text"""]      )

						# More info about loading csv files in the documentation: https://huggingface.co/docs/datasets/loading_datasets.html?highlight=csv#csv-files

						# Then split the documents into passages of 100 words
						A__						:  Union[str, Any]		  =  dataset.map(UpperCAmelCase__,						batched=UpperCAmelCase__,						num_proc=processing_args.num_proc      )

						# And compute the embeddings
						A__						:  str		  =  DPRContextEncoder.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name      ).to(device=UpperCAmelCase__      )
						A__						:  str		  =  DPRContextEncoderTokenizerFast.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name      )
						A__						:  Union[str, Any]		  =  Features(
						    {"""text""": Value("""string"""      ), """title""": Value("""string"""      ), """embeddings""": Sequence(Value("""float32"""      )      )}      )  # optional, save as float32 instead of float64 to save space
						A__						:  List[str]		  =  dataset.map(
						    partial(UpperCAmelCase__,						ctx_encoder=UpperCAmelCase__,						ctx_tokenizer=UpperCAmelCase__      ),						batched=UpperCAmelCase__,						batch_size=processing_args.batch_size,						features=UpperCAmelCase__,						)

						# And finally save your dataset
						A__						:  int		  =  os.path.join(rag_example_args.output_dir,						"""my_knowledge_dataset"""      )
						dataset.save_to_disk(UpperCAmelCase__      )
						# from datasets import load_from_disk
						# dataset = load_from_disk(passages_path)  # to reload the dataset

						######################################
						logger.info("""Step 2 - Index the dataset"""      )
						######################################

						# Let's use the Faiss implementation of HNSW for fast approximate nearest neighbor search
						A__						:  List[Any]		  =  faiss.IndexHNSWFlat(index_hnsw_args.d,						index_hnsw_args.m,						faiss.METRIC_INNER_PRODUCT      )
						dataset.add_faiss_index("""embeddings""",						custom_index=UpperCAmelCase__      )

						# And save the index
						A__						:  Optional[int]		  =  os.path.join(rag_example_args.output_dir,						"""my_knowledge_dataset_hnsw_index.faiss"""      )
						dataset.get_index("""embeddings"""      ).save(UpperCAmelCase__      )
						# dataset.load_faiss_index("embeddings", index_path)  # to reload the index






@dataclass
class 							__SCREAMING_SNAKE_CASE						:
							snake_case_     =							field(
							    default=str(Path(UpperCamelCase		).parent / 'test_run' / 'dummy-kb' / 'my_knowledge_dataset.csv'		)       ,		metadata={'help': 'Path to a tab-separated csv file with columns \'title\' and \'text\''}       ,		)
							snake_case_     =							field(
							    default=UpperCamelCase       ,		metadata={'help': 'Question that is passed as input to RAG. Default is \'What does Moses\' rod turn into ?\'.'}       ,		)
							snake_case_     =							field(
							    default='facebook/rag-sequence-nq'       ,		metadata={'help': 'The RAG model to use. Either \'facebook/rag-sequence-nq\' or \'facebook/rag-token-nq\''}       ,		)
							snake_case_     =							field(
							    default='facebook/dpr-ctx_encoder-multiset-base'       ,		metadata={
							        'help': (
							            'The DPR context encoder model to use. Either \'facebook/dpr-ctx_encoder-single-nq-base\' or'
							            ' \'facebook/dpr-ctx_encoder-multiset-base\''
							        )
							    }       ,		)
							snake_case_     =							field(
							    default=str(Path(UpperCamelCase		).parent / 'test_run' / 'dummy-kb'		)       ,		metadata={'help': 'Path to a directory where the dataset passages and the index will be saved'}       ,		)






@dataclass
class 							__SCREAMING_SNAKE_CASE						:
							snake_case_     =							field(
							    default=UpperCamelCase       ,		metadata={
							        'help': 'The number of processes to use to split the documents into passages. Default is single process.'
							    }       ,		)
							snake_case_     =							field(
							    default=16       ,		metadata={
							        'help': 'The batch size to use when computing the passages embeddings using the DPR context encoder.'
							    }       ,		)






@dataclass
class 							__SCREAMING_SNAKE_CASE						:
							snake_case_     =							field(
							    default=768       ,		metadata={'help': 'The dimension of the embeddings to pass to the HNSW Faiss index.'}       ,		)
							snake_case_     =							field(
							    default=128       ,		metadata={
							        'help': (
							            'The number of bi-directional links created for every new element during the HNSW index construction.'
							        )
							    }       ,		)


if __name__ == "__main__":
						logging.basicConfig(level=logging.WARNING)
						logger.setLevel(logging.INFO)

						A_    =							HfArgumentParser((RagExampleArguments, ProcessingArguments, IndexHnswArguments))
						A_    ,	A_    ,	A_    =							parser.parse_args_into_dataclasses()
						with TemporaryDirectory() as tmp_dir:
												A_    =							rag_example_args.output_dir or tmp_dir
												main(rag_example_args, processing_args, index_hnsw_args)














"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

A_    =							logging.get_logger(__name__)


A_    =							{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

A_    =							{
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

A_    =							{'''facebook/blenderbot-3B''': 128}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      _lowerCAmelCase		(							)		->Any:
						A__						:  Any		  =  (
						    list(range(ord("""!"""      ),						ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      ),						ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      ),						ord("""ÿ"""      ) + 1      )      )
						)
						A__						:  Optional[int]		  =  bs[:]
						A__						:  Union[str, Any]		  =  0
						for b in range(2**8      ):
												if b not in bs:
																		bs.append(UpperCAmelCase__      )
																		cs.append(2**8 + n      )
																		n += 1
						A__						:  Optional[int]		  =  [chr(UpperCAmelCase__      ) for n in cs]
						return dict(zip(UpperCAmelCase__,						UpperCAmelCase__      )      )




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Dict      )		->Tuple:
						A__						:  Any		  =  set()
						A__						:  Optional[int]		  =  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)      )
												A__						:  Dict		  =  char
						return pairs






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							VOCAB_FILES_NAMES
							snake_case_     =							PRETRAINED_VOCAB_FILES_MAP
							snake_case_     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_     =							['input_ids', 'attention_mask']


							def __init__(			self      :						int			,    snake_case      :						str			,    snake_case      :						Optional[Any]			,    snake_case      :						str="replace"			,    snake_case      :						Tuple="<s>"			,    snake_case      :						Dict="</s>"			,    snake_case      :						Optional[Any]="</s>"			,    snake_case      :						int="<s>"			,    snake_case      :						Dict="<unk>"			,    snake_case      :						List[Any]="<pad>"			,    snake_case      :						Union[str, Any]="<mask>"			,    snake_case      :						Optional[Any]=False			,    **snake_case      :						Union[str, Any]			,    ):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else bos_token
													A__						:  Dict		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else eos_token
													A__						:  List[str]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else sep_token
													A__						:  Tuple		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else cls_token
													A__						:  Dict		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else unk_token
													A__						:  int		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else pad_token

													# Mask token behave like a normal word, i.e. include the space before it
													A__						:  str		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else mask_token

													super().__init__(
													    errors=snake_case			,    bos_token=snake_case			,    eos_token=snake_case			,    unk_token=snake_case			,    sep_token=snake_case			,    cls_token=snake_case			,    pad_token=snake_case			,    mask_token=snake_case			,    add_prefix_space=snake_case			,    **snake_case			,    )

													with open(snake_case			,    encoding="""utf-8"""							) as vocab_handle:
																			A__						:  Dict		  =  json.load(snake_case							)
													A__						:  Tuple		  =  {v: k for k, v in self.encoder.items()}
													A__						:  Union[str, Any]		  =  errors  # how to handle errors in decoding
													A__						:  int		  =  bytes_to_unicode()
													A__						:  str		  =  {v: k for k, v in self.byte_encoder.items()}
													with open(snake_case			,    encoding="""utf-8"""							) as merges_handle:
																			A__						:  Tuple		  =  merges_handle.read().split("""\n"""							)[1:-1]
													A__						:  Optional[Any]		  =  [tuple(merge.split()							) for merge in bpe_merges]
													A__						:  List[str]		  =  dict(zip(snake_case			,    range(len(snake_case							)							)							)							)
													A__						:  Union[str, Any]		  =  {}
													A__						:  int		  =  add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													A__						:  Optional[Any]		  =  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)


							@property
							# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													return len(self.encoder							)


							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													return dict(self.encoder			,    **self.added_tokens_encoder							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						Optional[Any]							):
													'''simple docstring'''





													if token in self.cache:
																			return self.cache[token]
													A__						:  Tuple		  =  tuple(snake_case							)
													A__						:  Union[str, Any]		  =  get_pairs(snake_case							)

													if not pairs:
																			return token

													while True:
																			A__						:  Any		  =  min(snake_case			,    key=lambda snake_case							: self.bpe_ranks.get(snake_case			,    float("""inf"""							)							)							)
																			if bigram not in self.bpe_ranks:
																									break
																			A__  ,  A__						:  Optional[int]		  =  bigram
																			A__						:  Tuple		  =  []
																			A__						:  Dict		  =  0
																			while i < len(snake_case							):
																									try:
																															A__						:  int		  =  word.index(snake_case			,    snake_case							)
																									except ValueError:
																															new_word.extend(word[i:]							)
																															break
																									else:
																															new_word.extend(word[i:j]							)
																															A__						:  Optional[Any]		  =  j

																									if word[i] == first and i < len(snake_case							) - 1 and word[i + 1] == second:
																															new_word.append(first + second							)
																															i += 2
																									else:
																															new_word.append(word[i]							)
																															i += 1
																			A__						:  Optional[int]		  =  tuple(snake_case							)
																			A__						:  str		  =  new_word
																			if len(snake_case							) == 1:
																									break
																			else:
																									A__						:  List[str]		  =  get_pairs(snake_case							)
													A__						:  int		  =  """ """.join(snake_case							)
													A__						:  Optional[int]		  =  word
													return word


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						int							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  []
													for token in re.findall(self.pat			,    snake_case							):
																			A__						:  Optional[Any]		  =  """""".join(
																			    self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case							).split(""" """							)							)
													return bpe_tokens


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Union[str, Any]							):
													'''simple docstring'''





													return self.encoder.get(snake_case			,    self.encoder.get(self.unk_token							)							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]			,    snake_case      :						int							):
													'''simple docstring'''





													return self.decoder.get(snake_case							)


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						int							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  """""".join(snake_case							)
													A__						:  Dict		  =  bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""			,    errors=self.errors							)
													return text


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						str			,    snake_case      :						Optional[str] = None							):
													'''simple docstring'''





													if not os.path.isdir(snake_case							):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
																			return
													A__						:  Any		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
													A__						:  List[Any]		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

													with open(snake_case			,    """w"""			,    encoding="""utf-8"""							) as f:
																			f.write(json.dumps(self.encoder			,    indent=2			,    sort_keys=snake_case			,    ensure_ascii=snake_case							) + """\n"""							)

													A__						:  str		  =  0
													with open(snake_case			,    """w"""			,    encoding="""utf-8"""							) as writer:
																			writer.write("""#version: 0.2\n"""							)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,    key=lambda snake_case							: kv[1]							):
																									if index != token_index:
																															logger.warning(
																															    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																															    """ Please check that the tokenizer is not corrupted!"""							)
																															A__						:  List[str]		  =  token_index
																									writer.write(""" """.join(snake_case							) + """\n"""							)
																									index += 1

													return vocab_file, merge_file


							def 						_UpperCamelCase       (			self      :						Any			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None			,    snake_case      :						bool = False							):
													'''simple docstring'''





													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=snake_case			,    token_ids_a=snake_case			,    already_has_special_tokens=snake_case							)

													if token_ids_a is None:
																			return [1] + ([0] * len(snake_case							)) + [1]
													return [1] + ([0] * len(snake_case							)) + [1, 1] + ([0] * len(snake_case							)) + [1]


							def 						_UpperCamelCase       (			self      :						List[str]			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  [self.sep_token_id]
													A__						:  Optional[Any]		  =  [self.cls_token_id]

													if token_ids_a is None:
																			return len(cls + token_ids_a + sep							) * [0]
													return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


							def 						_UpperCamelCase       (			self      :						List[Any]			,    snake_case      :						Optional[int]			,    snake_case      :						List[Any]=False			,    **snake_case      :						Tuple							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  kwargs.pop("""add_prefix_space"""			,    self.add_prefix_space							)
													if (is_split_into_words or add_prefix_space) and (len(snake_case							) > 0 and not text[0].isspace()):
																			A__						:  int		  =  """ """ + text
													return (text, kwargs)


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													return token_ids_a + [self.eos_token_id]


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						"Conversation"							):
													'''simple docstring'''





													A__						:  str		  =  []
													for is_user, text in conversation.iter_texts():
																			if is_user:
																									# We need to space prefix as it's being done within blenderbot
																									inputs.append(""" """ + text							)
																			else:
																									# Generated responses should contain them already.
																									inputs.append(snake_case							)

													A__						:  Tuple		  =  """  """.join(snake_case							)
													A__						:  Dict		  =  self.encode(snake_case							)
													if len(snake_case							) > self.model_max_length:
																			A__						:  Optional[int]		  =  input_ids[-self.model_max_length :]
																			logger.warning(F'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'							)
													return input_ids






