changed to default settings

chatBot/common/llama.py

@@ -7,7 +7,7 @@ from langchain.embeddings import HuggingFaceInstructEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
 from pdf2image import convert_from_path
-from transformers import AutoTokenizer, TextStreamer, pipeline, AutoModelForCausalLM
+from transformers import AutoTokenizer, TextStreamer, pipeline
 from chatBot.common.pdfToText import loadLatestPdf
 from transformers import LlamaTokenizer
 from langchain.document_loaders import PyPDFLoader
@@ -28,27 +28,23 @@ texts = text_splitter.split_documents(data)
 db = Chroma.from_documents(texts, embeddings, persist_directory="db")
 
 model_name_or_path = "TheBloke/Llama-2-13B-chat-GPTQ"
-model_name_or_path = "TheBloke/Llama-2-13B-chat-GGUF"
 model_basename = "model"
 
 tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
 
-# model = AutoGPTQForCausalLM.from_quantized(
-#     model_name_or_path,
-#     device_map="auto",
-#     revision="gptq-4bit-128g-actorder_True",
-#     model_basename=model_basename,
-#     use_safetensors=True,
-#     trust_remote_code=True,
-#     inject_fused_attention=False,
-#     device=DEVICE,
-#     quantize_config=None,
-# )
-
-model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
-                                             device_map="auto",
-                                             trust_remote_code=False,
-                                             revision="main")
+model = AutoGPTQForCausalLM.from_quantized(
+    model_name_or_path,
+    device_map="auto",
+    revision="gptq-4bit-128g-actorder_True",
+    model_basename=model_basename,
+    use_safetensors=True,
+    trust_remote_code=True,
+    inject_fused_attention=False,
+    device=DEVICE,
+    quantize_config=None,
+)
+
+
 
 
 DEFAULT_SYSTEM_PROMPT = """