Spaces:
Sleeping
Sleeping
Commit
·
f57d5e5
1
Parent(s):
165d157
Update app.py
Browse files
app.py
CHANGED
@@ -17,7 +17,7 @@ import requests
|
|
17 |
from charset_normalizer import from_bytes
|
18 |
import zipfile
|
19 |
import tempfile
|
20 |
-
import
|
21 |
|
22 |
# Custom Exception Class
|
23 |
class GPUQuotaExceededError(Exception):
|
@@ -125,15 +125,15 @@ def get_model():
|
|
125 |
@spaces.GPU
|
126 |
def process_files(files):
|
127 |
if not files:
|
128 |
-
return "Please upload one or more.txt files.", ""
|
129 |
|
130 |
try:
|
131 |
if not initialize_model():
|
132 |
-
return "Failed to initialize the model. Please try again.", ""
|
133 |
|
134 |
valid_files = [f for f in files if f.name.lower().endswith('.txt')]
|
135 |
if not valid_files:
|
136 |
-
return "No.txt files found. Please upload valid.txt files.", ""
|
137 |
|
138 |
all_chunks = []
|
139 |
processed_files = 0
|
@@ -154,7 +154,7 @@ def process_files(files):
|
|
154 |
logger.error(f"Error processing file {file.name}: {str(e)}")
|
155 |
|
156 |
if not all_chunks:
|
157 |
-
return "No valid content found in the uploaded files.", ""
|
158 |
|
159 |
# Generate embeddings in batches
|
160 |
all_embeddings = []
|
@@ -164,7 +164,7 @@ def process_files(files):
|
|
164 |
embeddings = handle_gpu_operation(lambda: model.encode(batch))
|
165 |
all_embeddings.extend(embeddings)
|
166 |
else:
|
167 |
-
return "Model not initialized. Please check model initialization.", ""
|
168 |
|
169 |
# Save results to OUTPUTS_DIR
|
170 |
embeddings_path = os.path.join(OUTPUTS_DIR, "embeddings.npy")
|
@@ -176,13 +176,12 @@ def process_files(files):
|
|
176 |
|
177 |
return (
|
178 |
f"Successfully processed {processed_files} files. Generated {len(all_embeddings)} embeddings from {len(all_chunks)} chunks.",
|
179 |
-
"",
|
180 |
""
|
181 |
)
|
182 |
|
183 |
except Exception as e:
|
184 |
logger.error(f"Processing failed: {str(e)}")
|
185 |
-
return f"Error processing files: {str(e)}", ""
|
186 |
|
187 |
@spaces.GPU
|
188 |
def semantic_search(query, top_k=5):
|
@@ -194,13 +193,6 @@ def semantic_search(query, top_k=5):
|
|
194 |
# Load saved embeddings and chunks from OUTPUTS_DIR
|
195 |
embeddings_file = os.path.join(OUTPUTS_DIR, "embeddings.npy")
|
196 |
chunks_file = os.path.join(OUTPUTS_DIR, "chunks.txt")
|
197 |
-
|
198 |
-
logger.info(f"Checking for embeddings file: {embeddings_file}")
|
199 |
-
logger.info(f"Checking for chunks file: {chunks_file}")
|
200 |
-
|
201 |
-
if not os.path.exists(embeddings_file) or not os.path.exists(chunks_file):
|
202 |
-
return "Embeddings or chunks not found. Please generate embeddings first."
|
203 |
-
|
204 |
stored_embeddings = np.load(embeddings_file)
|
205 |
with open(chunks_file, "r", encoding="utf-8") as f:
|
206 |
chunks = f.read().split("\n===CHUNK_SEPARATOR===\n")
|
@@ -233,31 +225,17 @@ def search_and_format(query, num_results):
|
|
233 |
return "Please enter a search query"
|
234 |
return semantic_search(query, top_k=num_results)
|
235 |
|
236 |
-
def
|
237 |
try:
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
return "
|
244 |
-
|
245 |
-
def download_results():
|
246 |
-
required_files = ["embeddings.npy", "chunks.txt"]
|
247 |
-
missing = [f for f in required_files if not os.path.exists(os.path.join(OUTPUTS_DIR, f))]
|
248 |
-
if missing:
|
249 |
-
logger.error(f"Missing files: {missing}")
|
250 |
-
return None
|
251 |
-
try:
|
252 |
-
zip_path = os.path.join(OUTPUTS_DIR, "results.zip")
|
253 |
-
with zipfile.ZipFile(zip_path, 'w') as zipf:
|
254 |
-
for file in required_files:
|
255 |
-
file_path = os.path.join(OUTPUTS_DIR, file)
|
256 |
-
zipf.write(file_path, file)
|
257 |
-
return zip_path
|
258 |
except Exception as e:
|
259 |
-
logger.error(f"Error
|
260 |
-
return
|
261 |
|
262 |
def create_gradio_interface():
|
263 |
with gr.Blocks() as demo:
|
@@ -278,7 +256,7 @@ def create_gradio_interface():
|
|
278 |
process_button.click(
|
279 |
fn=process_files,
|
280 |
inputs=[file_input],
|
281 |
-
outputs=[output_text, error_box
|
282 |
)
|
283 |
|
284 |
with gr.Tab("Search"):
|
@@ -305,18 +283,12 @@ def create_gradio_interface():
|
|
305 |
outputs=results_output
|
306 |
)
|
307 |
|
308 |
-
|
309 |
-
|
310 |
-
|
311 |
-
|
312 |
-
|
313 |
-
|
314 |
-
with gr.Tab("Outputs"):
|
315 |
-
browse_button = gr.Button(" Browse Outputs")
|
316 |
-
browse_button.click(
|
317 |
-
fn=browse_outputs,
|
318 |
-
outputs=[gr.Textbox(label="Browse Status")]
|
319 |
-
)
|
320 |
|
321 |
return demo
|
322 |
|
|
|
17 |
from charset_normalizer import from_bytes
|
18 |
import zipfile
|
19 |
import tempfile
|
20 |
+
import shutil
|
21 |
|
22 |
# Custom Exception Class
|
23 |
class GPUQuotaExceededError(Exception):
|
|
|
125 |
@spaces.GPU
|
126 |
def process_files(files):
|
127 |
if not files:
|
128 |
+
return "Please upload one or more.txt files.", ""
|
129 |
|
130 |
try:
|
131 |
if not initialize_model():
|
132 |
+
return "Failed to initialize the model. Please try again.", ""
|
133 |
|
134 |
valid_files = [f for f in files if f.name.lower().endswith('.txt')]
|
135 |
if not valid_files:
|
136 |
+
return "No.txt files found. Please upload valid.txt files.", ""
|
137 |
|
138 |
all_chunks = []
|
139 |
processed_files = 0
|
|
|
154 |
logger.error(f"Error processing file {file.name}: {str(e)}")
|
155 |
|
156 |
if not all_chunks:
|
157 |
+
return "No valid content found in the uploaded files.", ""
|
158 |
|
159 |
# Generate embeddings in batches
|
160 |
all_embeddings = []
|
|
|
164 |
embeddings = handle_gpu_operation(lambda: model.encode(batch))
|
165 |
all_embeddings.extend(embeddings)
|
166 |
else:
|
167 |
+
return "Model not initialized. Please check model initialization.", ""
|
168 |
|
169 |
# Save results to OUTPUTS_DIR
|
170 |
embeddings_path = os.path.join(OUTPUTS_DIR, "embeddings.npy")
|
|
|
176 |
|
177 |
return (
|
178 |
f"Successfully processed {processed_files} files. Generated {len(all_embeddings)} embeddings from {len(all_chunks)} chunks.",
|
|
|
179 |
""
|
180 |
)
|
181 |
|
182 |
except Exception as e:
|
183 |
logger.error(f"Processing failed: {str(e)}")
|
184 |
+
return f"Error processing files: {str(e)}", ""
|
185 |
|
186 |
@spaces.GPU
|
187 |
def semantic_search(query, top_k=5):
|
|
|
193 |
# Load saved embeddings and chunks from OUTPUTS_DIR
|
194 |
embeddings_file = os.path.join(OUTPUTS_DIR, "embeddings.npy")
|
195 |
chunks_file = os.path.join(OUTPUTS_DIR, "chunks.txt")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
196 |
stored_embeddings = np.load(embeddings_file)
|
197 |
with open(chunks_file, "r", encoding="utf-8") as f:
|
198 |
chunks = f.read().split("\n===CHUNK_SEPARATOR===\n")
|
|
|
225 |
return "Please enter a search query"
|
226 |
return semantic_search(query, top_k=num_results)
|
227 |
|
228 |
+
def copy_embeddings_to_workspace():
|
229 |
try:
|
230 |
+
embeddings_path = os.path.join(OUTPUTS_DIR, "embeddings.npy")
|
231 |
+
chunks_path = os.path.join(OUTPUTS_DIR, "chunks.txt")
|
232 |
+
workspace_dir = os.getcwd()
|
233 |
+
shutil.copy(embeddings_path, workspace_dir)
|
234 |
+
shutil.copy(chunks_path, workspace_dir)
|
235 |
+
return "Embeddings copied to workspace directory."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
236 |
except Exception as e:
|
237 |
+
logger.error(f"Error copying embeddings: {str(e)}")
|
238 |
+
return f"Error copying embeddings: {str(e)}"
|
239 |
|
240 |
def create_gradio_interface():
|
241 |
with gr.Blocks() as demo:
|
|
|
256 |
process_button.click(
|
257 |
fn=process_files,
|
258 |
inputs=[file_input],
|
259 |
+
outputs=[output_text, error_box]
|
260 |
)
|
261 |
|
262 |
with gr.Tab("Search"):
|
|
|
283 |
outputs=results_output
|
284 |
)
|
285 |
|
286 |
+
copy_button = gr.Button("Copy Embeddings to Workspace")
|
287 |
+
copy_output = gr.Textbox(label="Copy Status")
|
288 |
+
copy_button.click(
|
289 |
+
fn=copy_embeddings_to_workspace,
|
290 |
+
outputs=[copy_output]
|
291 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
|
293 |
return demo
|
294 |
|