| callbacks: | |
| rollout_lh: | |
| tasks: | |
| _target_: calvin_env.envs.tasks.Tasks | |
| tasks: | |
| rotate_red_block_right: | |
| - rotate_object | |
| - block_red | |
| - -60 | |
| rotate_red_block_left: | |
| - rotate_object | |
| - block_red | |
| - 60 | |
| rotate_blue_block_right: | |
| - rotate_object | |
| - block_blue | |
| - -60 | |
| rotate_blue_block_left: | |
| - rotate_object | |
| - block_blue | |
| - 60 | |
| rotate_pink_block_right: | |
| - rotate_object | |
| - block_pink | |
| - -60 | |
| rotate_pink_block_left: | |
| - rotate_object | |
| - block_pink | |
| - 60 | |
| push_red_block_right: | |
| - push_object | |
| - block_red | |
| - 0.1 | |
| - 0 | |
| push_red_block_left: | |
| - push_object | |
| - block_red | |
| - -0.1 | |
| - 0 | |
| push_blue_block_right: | |
| - push_object | |
| - block_blue | |
| - 0.1 | |
| - 0 | |
| push_blue_block_left: | |
| - push_object | |
| - block_blue | |
| - -0.1 | |
| - 0 | |
| push_pink_block_right: | |
| - push_object | |
| - block_pink | |
| - 0.1 | |
| - 0 | |
| push_pink_block_left: | |
| - push_object | |
| - block_pink | |
| - -0.1 | |
| - 0 | |
| move_slider_left: | |
| - move_door_rel | |
| - base__slide | |
| - 0.15 | |
| move_slider_right: | |
| - move_door_rel | |
| - base__slide | |
| - -0.15 | |
| open_drawer: | |
| - move_door_rel | |
| - base__drawer | |
| - 0.12 | |
| close_drawer: | |
| - move_door_rel | |
| - base__drawer | |
| - -0.12 | |
| lift_red_block_table: | |
| - lift_object | |
| - block_red | |
| - 0.05 | |
| - table | |
| - base_link | |
| lift_red_block_slider: | |
| - lift_object | |
| - block_red | |
| - 0.03 | |
| - table | |
| - plank_link | |
| lift_red_block_drawer: | |
| - lift_object | |
| - block_red | |
| - 0.05 | |
| - table | |
| - drawer_link | |
| lift_blue_block_table: | |
| - lift_object | |
| - block_blue | |
| - 0.05 | |
| - table | |
| - base_link | |
| lift_blue_block_slider: | |
| - lift_object | |
| - block_blue | |
| - 0.03 | |
| - table | |
| - plank_link | |
| lift_blue_block_drawer: | |
| - lift_object | |
| - block_blue | |
| - 0.05 | |
| - table | |
| - drawer_link | |
| lift_pink_block_table: | |
| - lift_object | |
| - block_pink | |
| - 0.05 | |
| - table | |
| - base_link | |
| lift_pink_block_slider: | |
| - lift_object | |
| - block_pink | |
| - 0.03 | |
| - table | |
| - plank_link | |
| lift_pink_block_drawer: | |
| - lift_object | |
| - block_pink | |
| - 0.05 | |
| - table | |
| - drawer_link | |
| place_in_slider: | |
| - place_object | |
| - table | |
| - plank_link | |
| place_in_drawer: | |
| - place_object | |
| - table | |
| - drawer_link | |
| stack_block: | |
| - stack_objects | |
| unstack_block: | |
| - unstack_objects | |
| turn_on_lightbulb: | |
| - toggle_light | |
| - lightbulb | |
| - 0 | |
| - 1 | |
| turn_off_lightbulb: | |
| - toggle_light | |
| - lightbulb | |
| - 1 | |
| - 0 | |
| turn_on_led: | |
| - toggle_light | |
| - led | |
| - 0 | |
| - 1 | |
| turn_off_led: | |
| - toggle_light | |
| - led | |
| - 1 | |
| - 0 | |
| push_into_drawer: | |
| - push_object_into | |
| - - block_red | |
| - block_blue | |
| - block_pink | |
| - table | |
| - base_link | |
| - table | |
| - drawer_link | |
| val_annotations: | |
| rotate_red_block_right: | |
| - take the red block and rotate it to the right | |
| rotate_red_block_left: | |
| - take the red block and rotate it to the left | |
| rotate_blue_block_right: | |
| - take the blue block and rotate it to the right | |
| rotate_blue_block_left: | |
| - take the blue block and rotate it to the left | |
| rotate_pink_block_right: | |
| - take the pink block and rotate it to the right | |
| rotate_pink_block_left: | |
| - take the pink block and rotate it to the left | |
| push_red_block_right: | |
| - go push the red block right | |
| push_red_block_left: | |
| - go push the red block left | |
| push_blue_block_right: | |
| - go push the blue block right | |
| push_blue_block_left: | |
| - go push the blue block left | |
| push_pink_block_right: | |
| - go push the pink block right | |
| push_pink_block_left: | |
| - go push the pink block left | |
| move_slider_left: | |
| - push the sliding door to the left side | |
| move_slider_right: | |
| - push the sliding door to the right side | |
| open_drawer: | |
| - pull the handle to open the drawer | |
| close_drawer: | |
| - push the handle to close the drawer | |
| lift_red_block_table: | |
| - grasp and lift the red block | |
| lift_blue_block_table: | |
| - grasp and lift the blue block | |
| lift_pink_block_table: | |
| - grasp and lift the pink block | |
| lift_red_block_slider: | |
| - lift the red block from the sliding cabinet | |
| lift_blue_block_slider: | |
| - lift the blue block from the sliding cabinet | |
| lift_pink_block_slider: | |
| - lift the pink block from the sliding cabinet | |
| lift_red_block_drawer: | |
| - Take the red block from the drawer | |
| lift_blue_block_drawer: | |
| - Take the blue block from the drawer | |
| lift_pink_block_drawer: | |
| - Take the pink block from the drawer | |
| place_in_slider: | |
| - store the grasped block in the sliding cabinet | |
| place_in_drawer: | |
| - store the grasped block in the drawer | |
| push_into_drawer: | |
| - slide the block that it falls into the drawer | |
| stack_block: | |
| - stack the grasped block | |
| unstack_block: | |
| - remove the stacked block | |
| turn_on_lightbulb: | |
| - use the switch to turn on the light bulb | |
| turn_off_lightbulb: | |
| - use the switch to turn off the light bulb | |
| turn_on_led: | |
| - press the button to turn on the led light | |
| turn_off_led: | |
| - press the button to turn off the led light | |
| _target_: flower.rollout.rollout_long_horizon.RolloutLongHorizon | |
| _recursive_: false | |
| env_cfg: | |
| _target_: flower.wrappers.hulc_wrapper.HulcWrapper | |
| skip_epochs: ${rollout_lh_skip_epochs} | |
| rollout_freq: 5 | |
| num_videos: 0 | |
| num_sequences: 1000 | |
| replan_freq: 30 | |
| ep_len: 360 | |
| empty_cache: false | |
| log_video_to_file: false | |
| save_dir: ./videos | |
| lang_folder: ${lang_folder} | |
| debug: false | |
| ema: | |
| _target_: flower.callbacks.ema.EMA | |
| decay: 0.999 | |
| start_step: 0 | |
| save_ema_weights_in_callback_state: true | |
| evaluate_ema_weights_instead: true | |
| power: 0.6666666666666666 | |
| inv_gamma: 1.0 | |
| min_value: 0.0 | |
| max_value: 0.9999 | |
| checkpoint: | |
| _target_: pytorch_lightning.callbacks.ModelCheckpoint | |
| save_top_k: 1 | |
| verbose: true | |
| monitor: eval_lh/avg_seq_len | |
| mode: max | |
| dirpath: saved_models | |
| filename: '{epoch:02d}_{eval_lh/avg_seq_len:.2f}' | |
| every_n_epochs: ${callbacks.rollout_lh.rollout_freq} | |
| datamodule: | |
| transforms: | |
| train: | |
| rgb_static: | |
| - _target_: torchvision.transforms.Resize | |
| size: 224 | |
| antialias: true | |
| - _target_: flower.utils.transforms.RandomShiftsAug | |
| pad: 10 | |
| - _target_: flower.utils.transforms.ScaleImageTensor | |
| - _target_: torchvision.transforms.Normalize | |
| mean: | |
| - 0.48145466 | |
| - 0.4578275 | |
| - 0.40821073 | |
| std: | |
| - 0.26862954 | |
| - 0.26130258 | |
| - 0.27577711 | |
| rgb_gripper: | |
| - _target_: torchvision.transforms.Resize | |
| size: 224 | |
| antialias: true | |
| - _target_: flower.utils.transforms.RandomShiftsAug | |
| pad: 4 | |
| - _target_: flower.utils.transforms.ScaleImageTensor | |
| - _target_: torchvision.transforms.Normalize | |
| mean: | |
| - 0.48145466 | |
| - 0.4578275 | |
| - 0.40821073 | |
| std: | |
| - 0.26862954 | |
| - 0.26130258 | |
| - 0.27577711 | |
| val: | |
| rgb_static: | |
| - _target_: torchvision.transforms.Resize | |
| size: 224 | |
| antialias: true | |
| - _target_: flower.utils.transforms.ScaleImageTensor | |
| - _target_: torchvision.transforms.Normalize | |
| mean: | |
| - 0.48145466 | |
| - 0.4578275 | |
| - 0.40821073 | |
| std: | |
| - 0.26862954 | |
| - 0.26130258 | |
| - 0.27577711 | |
| rgb_gripper: | |
| - _target_: torchvision.transforms.Resize | |
| size: 224 | |
| antialias: true | |
| - _target_: flower.utils.transforms.ScaleImageTensor | |
| - _target_: torchvision.transforms.Normalize | |
| mean: | |
| - 0.48145466 | |
| - 0.4578275 | |
| - 0.40821073 | |
| std: | |
| - 0.26862954 | |
| - 0.26130258 | |
| - 0.27577711 | |
| _target_: flower.datasets.hulc_data_module.HulcDataModule | |
| _recursive_: false | |
| root_data_dir: ${root_data_dir} | |
| action_space: 7 | |
| action_max: | |
| - 1.0 | |
| - 1.0 | |
| - 1.0 | |
| - 1.0 | |
| - 1.0 | |
| - 1.0 | |
| - 1.0 | |
| action_min: | |
| - -1.0 | |
| - -1.0 | |
| - -1.0 | |
| - -1.0 | |
| - -1.0 | |
| - -1.0 | |
| - -1 | |
| shuffle_val: false | |
| observation_space: | |
| rgb_obs: | |
| - rgb_static | |
| - rgb_gripper | |
| depth_obs: [] | |
| state_obs: | |
| - robot_obs | |
| actions: | |
| - rel_actions | |
| language: | |
| - language | |
| proprioception_dims: | |
| n_state_obs: 8 | |
| keep_indices: | |
| - - 0 | |
| - 7 | |
| - - 14 | |
| - 15 | |
| robot_orientation_idx: | |
| - 3 | |
| - 6 | |
| normalize: true | |
| normalize_robot_orientation: true | |
| datasets: | |
| lang_dataset: | |
| _target_: flower.datasets.disk_dataset.ExtendedDiskDataset | |
| key: lang | |
| save_format: npz | |
| batch_size: ${batch_size} | |
| min_window_size: ${act_seq_len} | |
| max_window_size: ${act_seq_len} | |
| proprio_state: ${datamodule.proprioception_dims} | |
| obs_space: ${datamodule.observation_space} | |
| skip_frames: 1 | |
| pad: false | |
| lang_folder: ${lang_folder} | |
| aux_lang_loss_window: 8 | |
| num_workers: ${num_workers} | |
| action_seq_len: ${act_seq_len} | |
| obs_seq_len: ${obs_seq_len} | |
| future_range: 1 | |
| use_extracted_rel_actions: ${use_extracted_rel_actions} | |
| model: | |
| _target_: flower.models.flower.FLOWERVLA | |
| _recursive_: false | |
| vlm_path: microsoft/Florence-2-large | |
| freeze_florence: false | |
| freeze_vision_tower: false | |
| vlm_prompt_style: default | |
| token_dropout: 0.1 | |
| multistep: ${multistep} | |
| num_sampling_steps: 4 | |
| lowdim_obs_dim: 7 | |
| action_dim: 7 | |
| act_window_size: 10 | |
| load_pretrained: true | |
| pretrained_model_path: /home/hk-project-sustainebot/ft4740/code/flower_vla/logs/runs/2025-02-19/11-15-41/200000_model_weights.pt | |
| use_second_view: true | |
| second_view_key: image_wrist | |
| action_type_adaln: true | |
| use_causal_attention: true | |
| use_cross_attn: true | |
| use_adaln_cond: false | |
| use_readout_token: false | |
| use_proprio: false | |
| return_act_chunk: false | |
| sampling_type: uniform | |
| dit_dim: 1024 | |
| n_heads: 16 | |
| n_layers: 18 | |
| attn_pdrop: 0.1 | |
| resid_pdrop: 0.1 | |
| mlp_pdrop: 0.1 | |
| use_rope: true | |
| use_nope: false | |
| query_seq_len: 100 | |
| rope_theta: 32.0 | |
| optimizer_type: adamw | |
| optimizer: | |
| _target_: torch.optim.AdamW | |
| transformer_weight_decay: 0.05 | |
| learning_rate: 2.0e-05 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| lr_scheduler: | |
| lr_scheduler: | |
| init_lr: 2.0e-05 | |
| init_lr_scale: 0.1 | |
| final_lr_scale: 0.1 | |
| total_steps: 50000 | |
| phase_ratio: (0.05, 0.1, 0.85) | |
| lr: 5.0e-05 | |
| root_data_dir: /hkfs/work/workspace/scratch/ft4740-play3/task_ABC_D | |
| lang_folder: lang_clip_resnet50 | |
| log_dir: ./logs | |
| slurm: false | |
| seed: 242 | |
| device: cuda | |
| batch_size: 8 | |
| devices: 4 | |
| act_dim: 7 | |
| proprio_dims: 7 | |
| obs_dim: 512 | |
| goal_dim: 512 | |
| obs_seq_len: 1 | |
| act_seq_len: 10 | |
| multistep: 10 | |
| p_last_state: 0 | |
| gen_img_res: 112 | |
| max_epochs: 35 | |
| rollout_lh_skip_epochs: 9 | |
| num_workers: 12 | |
| benchmark_name: calvin_abc | |
| use_extracted_rel_actions: true | |
| trainer: | |
| devices: ${devices} | |
| precision: bf16-mixed | |
| max_epochs: ${max_epochs} | |
| sync_batchnorm: true | |
| accelerator: gpu | |
| strategy: ddp | |
| limit_train_batches: 1000 | |
| limit_val_batches: 4 | |
| logger: | |
| _target_: pytorch_lightning.loggers.WandbLogger | |
| save_dir: . | |
| name: logger | |
| group: mode | |
| log_model: false | |
| project: ${benchmark_name} | |
| entity: bennoq | |
| id: ??? | |