added audio transcription

app.py

@@ -6,8 +6,9 @@ from fastai.vision.all import load_learner
 import time
 import base64
 from deepface import DeepFace
-import torchaudio, torch
-import subprocess
+import torchaudio
+import moviepy.editor as mp
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
 
 # import pathlib
 # temp = pathlib.PosixPath
@@ -22,6 +23,32 @@ backends = [
   'mediapipe'
 ]
 
+def getTranscription(path):
+    # Insert Local Video File Path
+    clip = mp.VideoFileClip(path)
+
+    # Insert Local Audio File Path
+    clip.audio.write_audiofile(r"audio.wav")
+    
+    waveform, sample_rate = torchaudio.load("audio.wav")
+    waveform, sample_rate
+    
+    waveform, sample_rate = torchaudio.load("audio.wav")
+    resampler = torchaudio.transforms.Resample(sample_rate, 16000)
+    waveform = resampler(waveform)[0]
+    
+    processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
+    model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
+    model.config.forced_decoder_ids = None
+    
+    input_features = processor(waveform.squeeze(dim=0), return_tensors="pt").input_features 
+    predicted_ids = model.generate(input_features)
+
+    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
+    
+    return transcription[0]
+    
+
 model = load_learner("gaze-recognizer-v3.pkl")
 
 def video_processing(video_file, encoded_video):
@@ -45,19 +72,8 @@ def video_processing(video_file, encoded_video):
 
     start_time = time.time()
 
-    subprocess.call(["ffmpeg", "-i", 'temp_video.mp4', "audio.wav"])
-    waveform, sample_rate = torchaudio.load("audio.wav")
-    waveform, sample_rate
-
-    waveform, sample_rate = torchaudio.load("audio.wav")
-    resampler = torchaudio.transforms.Resample(sample_rate, 16000)
-    waveform = resampler(waveform)[0]
-
-    input_features = processor(waveform.squeeze(dim=0), return_tensors="pt").input_features 
-    predicted_ids = model.generate(input_features)
-    
-    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
-    print(transcription[0])
+    transcription = getTranscription(video_file)
+    print(transcription)
     
     video_capture = cv2.VideoCapture(video_file)
     on_camera = 0