import os import cv2 import numpy as np import tempfile from tqdm import tqdm from subprocess import call import trimesh from scipy.io import wavfile import librosa class Struct(object): def __init__(self, **kwargs): for key, val in kwargs.items(): setattr(self, key, val) def add_image_text(img, text, color=(0,0,255), w=800, h=800): font = cv2.FONT_HERSHEY_SIMPLEX img = np.require(img, dtype='f4', requirements=['O', 'W']) img.flags.writeable = True img1 = img.copy() img1 = cv2.putText(img1, '%s' % (text), (50, 100), font, 2, color, 2, 1) img1 = cv2.rectangle(img1, (0, 0), (w, h), color, thickness=3) return img1 class RenderTool: def __init__(self, out_path): path = os.path.join(os.getcwd(), 'visualise/smplx/SMPLX_NEUTRAL.npz') model_data = np.load(path, allow_pickle=True) data_struct = Struct(**model_data) self.f = data_struct.f # faces self.out_path = out_path if not os.path.exists(self.out_path): os.makedirs(self.out_path) def _render_sequences(self, cur_wav_file, v_list, j=-1, run_in_parallel=False): symbol = '/' print("Render {} {} sequence.".format(cur_wav_file.split(symbol)[-2], cur_wav_file.split(symbol)[-1])) directory = os.path.join(self.out_path, cur_wav_file.split(symbol)[2].split(symbol)[0]) if not os.path.exists(directory): os.makedirs(directory) if j == -1: video_fname = os.path.join(directory, '%s.mp4' % cur_wav_file.split(symbol)[-1].split('.')[-2].split(symbol)[-1]) elif j == -2: video_fname = os.path.join(directory, cur_wav_file.split(symbol)[-3]+'--%s.mp4' % cur_wav_file.split(symbol)[-1].split('.')[-2].split(symbol)[-1]) else: video_fname = os.path.join(directory, str(j)+'_%s.mp4' % cur_wav_file.split(symbol)[-1].split('.')[-2].split(symbol)[-1]) self._render_sequences_helper(video_fname, cur_wav_file, v_list) def _render_sequences_helper(self, video_fname, cur_wav_file, v_list): num_frames = v_list[0].shape[0] # Prepare output frames folder frames_dir = os.path.join(os.path.dirname(video_fname), "frames_tmp") os.makedirs(frames_dir, exist_ok=True) center = np.mean(v_list[0][0], axis=0) for i_frame in tqdm(range(num_frames)): cur_img_list = [] for i in range(len(v_list)): mesh = trimesh.Trimesh(vertices=v_list[i][i_frame], faces=self.f) scene = trimesh.Scene(mesh) png = scene.save_image(resolution=[800, 800], visible=True) img_array = np.asarray(bytearray(png), dtype=np.uint8) img = cv2.imdecode(img_array, cv2.IMREAD_COLOR) cur_img_list.append(img) if len(cur_img_list) == 1: final_img = cur_img_list[0] else: final_img = np.hstack(cur_img_list) frame_path = os.path.join(frames_dir, f"frame_{i_frame:04d}.png") cv2.imwrite(frame_path, final_img) # Create video from frames tmp_audio_file = tempfile.NamedTemporaryFile('w', suffix='.wav', dir=os.path.dirname(video_fname)) tmp_audio_file.close() audio, sr = librosa.load(cur_wav_file, sr=16000) wavfile.write(tmp_audio_file.name, sr, audio) tmp_video_file = tempfile.NamedTemporaryFile('w', suffix='.mp4', dir=os.path.dirname(video_fname)) tmp_video_file.close() cmd_frames_to_video = ( f"ffmpeg -framerate 30 -i {frames_dir}/frame_%04d.png -c:v libx264 -pix_fmt yuv420p {tmp_video_file.name}" ).split() call(cmd_frames_to_video) cmd_merge_audio = ( f"ffmpeg -i {tmp_video_file.name} -i {tmp_audio_file.name} -vcodec copy -acodec aac {video_fname}" ).split() call(cmd_merge_audio) # Cleanup os.remove(tmp_audio_file.name) os.remove(tmp_video_file.name) for f in os.listdir(frames_dir): os.remove(os.path.join(frames_dir, f)) os.rmdir(frames_dir)