Upload 5 files

checkpoints/240112_icl_audio2secc_vox2_cmlr/config.yaml

@@ -0,0 +1,86 @@
+accumulate_grad_batches: 1
+amp: false
+audio_type: hubert
+base_config:
+- ./audio2secc_vae.yaml
+batch_size: 4
+binarization_args:
+  with_coeff: true
+  with_hubert: true
+  with_mel: true
+binary_data_dir: data/binary/voxceleb2_audio2motion
+blink_mode: blink_unit
+clip_grad_norm: 0.5
+clip_grad_value: 0
+debug: false
+ds_name: Concat_voxceleb2_CMLR
+eval_max_batches: 10
+gen_dir_name: ''
+hidden_size: 256
+icl_model_type: icl_flow_matching
+infer_audio_source_name: ''
+infer_ckpt_steps: 40000
+infer_out_npy_name: ''
+init_from_ckpt: ''
+lambda_kl: 0.02
+lambda_kl_t1: 2000
+lambda_kl_t2: 2000
+lambda_l2_reg_exp: 0.0
+lambda_lap_exp: 0.0
+lambda_lap_exp_x1: 0.1
+lambda_mse_exp: 0.0
+lambda_mse_exp_x1: 0.1
+lambda_mse_icl: 1.0
+lambda_mse_lm2d: 0.0
+lambda_mse_lm3d: 0.0
+lambda_mse_lm3d_x1: 0.1
+lambda_sync_lm3d: 0.05
+load_ckpt: ''
+load_db_to_memory: false
+lr: 0.0005
+max_sentences_per_batch: 512
+max_tokens_per_batch: 20000
+max_updates: 4000000
+motion_type: exp
+num_ckpt_keep: 100
+num_sanity_val_steps: 5
+num_valid_plots: 1
+num_workers: 8
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.999
+print_nan_grads: false
+process_id: 0
+raw_data_dir: /home/tiger/datasets/raw/TH1KH_512
+ref_id_mode: first_frame
+resume_from_checkpoint: 0
+sample_min_length: 32
+save_best: false
+save_codes:
+- tasks
+- modules
+- egs
+save_gt: true
+scheduler: exponential
+seed: 9999
+smo_win_size: 5
+split_seed: 999
+syncnet_ckpt_dir: checkpoints/0904_syncnet/syncnet_hubert_vox2
+task_cls: tasks.os_avatar.icl_audio2secc_task.Audio2SECCTask
+tb_log_interval: 100
+total_process: 1
+use_aux_features: true
+use_aux_loss_on_x1: true
+use_eye_amp_embed: false
+use_flow: true
+use_fork: true
+use_kv_dataset: true
+use_mouth_amp_embed: true
+use_pitch: true
+val_check_interval: 2000
+valid_infer_interval: 2000
+valid_monitor_key: val_loss
+valid_monitor_mode: min
+warmup_updates: 1000
+weight_decay: 0
+work_dir: checkpoints/240112_audio2secc/icl_audio2secc_vox2_cmlr
+x_multiply: 16

checkpoints/240112_icl_audio2secc_vox2_cmlr/model_ckpt_steps_1856000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:367167db3b25fe07de9255871ce7813158551c4b00bfadd7334a671648924a2e
+size 462941609

checkpoints/mimictalk_orig/os_secc2plane_torso/config.yaml

@@ -0,0 +1,152 @@
+accumulate_grad_batches: 1
+add_ffhq_singe_disc: false
+also_update_decoder: false
+amp: false
+base_channel: 32768
+base_config:
+- ./secc_img2plane.yaml
+batch_size: 1
+binary_data_dir: data/binary/CelebV-HQ
+blur_fade_kimg: 20
+blur_init_sigma: 10
+blur_raw_target: true
+box_warp: 1
+ckpt_milestone_interval: 50000
+clip_grad_norm: 1.0
+clip_grad_value: 0
+cond_hid_dim: 32
+cond_out_dim: 16
+cond_type: idexp_lm3d_normalized
+debug: false
+density_reg_p_dist: 0.004
+disable_highreso_at_stage1: true
+disc_c_noise: 1.0
+disc_cond_mode: none
+ds_name: Concat_VFHQ_CelebVHQ_TH1KH_RAVDESS
+ema_interval: 400
+enable_rescale_plane_regulation: false
+eval_max_batches: 100
+ffhq_disc_inp_mode: eg3d_gen
+final_resolution: 512
+flipped_to_world_coord: true
+fuse_with_deform_source: false
+gen_cond_mode: none
+generator_condition_on_pose: true
+gpc_reg_fade_kimg: 1000
+gpc_reg_prob: 0.8
+group_size_for_mini_batch_std: 2
+htbsr_head_threshold: 1.0
+htbsr_head_weight_fuse_mode: v2
+img2plane_backbone_mode: composite
+img2plane_backbone_scale: standard
+init_from_ckpt: checkpoints/240118_os_secc2planes/os_secc2plane_pertubeBlink0.05_pertubeSECC0.05
+lam_occlusion_2_reg_l1: 0.0
+lam_occlusion_reg_l1: 0.0
+lam_occlusion_weights_entropy: 0.001
+lambda_G_adversarial_adv: 1.0
+lambda_G_supervise_adv: 1.0
+lambda_G_supervise_mse: 1.0
+lambda_G_supervise_mse_raw: 1.0
+lambda_density_reg: 0.25
+lambda_ffhq_mv_adv: 0.002
+lambda_gradient_penalty: 1.0
+lambda_mse: 1.0
+lambda_mse_depth: 0.0
+lambda_th1kh_mv_adv: 0.003
+lambda_weights_entropy: 0.01
+lambda_weights_l1: 0.1
+load_ckpt: ''
+lpips_mode: vgg19_v2
+lr_d: 0.0002
+lr_decay_interval: 5000
+lr_decay_rate: 0.95
+lr_g: 1.0e-05
+lr_lambda_pertube_secc: 0.01
+lr_mul_cano_img2plane: 1.0
+mapping_network_depth: 2
+max_channel: 512
+max_updates: 100000
+mimic_plane: false
+min_rescale_factor: 0.25
+motion_smo_win_size: 5
+neural_rendering_resolution: 128
+normalize_cond: false
+normalize_radius: false
+not_save_modules:
+- criterion_lpips
+- eg3d_model
+num_ckpt_keep: 1
+num_fp16_layers_in_discriminator: 4
+num_fp16_layers_in_generator: 0
+num_fp16_layers_in_super_resolution: 4
+num_samples_coarse: 48
+num_samples_fine: 48
+num_sanity_val_steps: 1
+num_valid_plots: 25
+num_workers: 8
+occlusion_fuse: true
+ones_ws_for_sr: true
+optimizer_adam_beta1_d: 0.0
+optimizer_adam_beta1_g: 0.0
+optimizer_adam_beta2_d: 0.99
+optimizer_adam_beta2_g: 0.99
+phase1_plane_fusion_mode: add
+pncc_cond_mode: cano_src_tgt
+pretrained_eg3d_ckpt: /mnt/bn/ailabrenyi/entries/yezhenhui/projects/GeneFace_private/checkpoints/0628_img2planes/eg3d_baseline_run2/model_ckpt_steps_100000.ckpt
+print_nan_grads: false
+process_id: 0
+processed_data_dir: data/processed/videos
+random_sample_pose: true
+raw_data_dir: /home/tiger/datasets/raw/FFHQ
+ray_far: auto
+ray_near: auto
+reg_interval_d: 16
+reg_interval_g: 4
+reg_interval_g_cond: 4
+reload_head_ckpt: ''
+resume_from_checkpoint: 0
+save_best: true
+save_codes:
+- tasks
+- modules
+- egs
+secc_pertube_mode: randn
+secc_pertube_randn_scale: 0.01
+secc_segformer_scale: b0
+seed: 9999
+seg_out_mode: head
+smo_win_size: 5
+split_seed: 999
+sr_type: vanilla
+start_adv_iters: 40000
+target_pertube_blink_secc_loss: 0.05
+target_pertube_secc_loss: 0.05
+task_cls: tasks.os_avatar.secc_img2plane_torso_task.SECC_Img2PlaneEG3D_TorsoTask
+tb_log_interval: 100
+torch_compile: true
+torso_kp_num: 4
+torso_model_version: v2
+torso_occlusion_reg_unmask_factor: 0.3
+torso_ref_segout_mode: torso
+total_process: 1
+triplane_depth: 3
+triplane_feature_type: trigrid_v2
+triplane_hid_dim: 32
+two_stage_training: true
+update_on_th1kh_samples: false
+update_src2src_interval: 4
+use_kv_dataset: true
+use_motion_smo_net: false
+use_mse: false
+use_th1kh_disc: false
+use_th1kh_mv_adv: false
+val_check_interval: 2000
+valid_infer_interval: 2000
+valid_monitor_key: val_loss
+valid_monitor_mode: min
+video_id: May
+w_dim: 512
+warmup_updates: 4000
+weight_fuse: true
+work_dir: checkpoints/240120_os_secc2planes_torso/os_secc2plane_torso_htbsrFusev2_htbsrThres1.0
+z_dim: 512

checkpoints/mimictalk_orig/os_secc2plane_torso/model_ckpt_steps_100000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:757c1b73d938da0ec3fba555a155a31ac803ddc8d343bba2a3c69845844b213a
+size 1414788463

checkpoints/pretrained_ckpts/mit_b0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df468f7f13c4186f25bd3e2caf09e4f927b5b5ac0abccac84011dae747d4c49c
+size 14331578