File size: 5,657 Bytes
54216bc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
"""
Evaluate a model on the egoschema dataset using LVNet (captions pre-generated)
Sample Run:
python3 LLM_stage.py \
--output-dir ego_base_link \
--captions data/ES_captions_gpt4o.jsonl \
--per-vid-captions 12 \
--gptmodel "gpt-4o" \
--temperature 0.0
"""
import argparse
import json
import os
from tqdm import tqdm
from src.run_gpt import run_gpt
# You may add multiple keys to run parallel calls
dict_api = {
"api_key": "ADD",
}
_PROMPT_TEMPLATE = (
"Here are descriptions of the video frames at specific times, noted in seconds."
"\n\n{Putdesc}.\n\nThe descriptions of the frames conclude. Think step-by-step"
" and I request your selection of the most appropriate response to the following"
" question\n\nQuestion:\n{Putquestion}\n\nOptions:\n{AllOptions}"
)
def eval_model(args):
# change split to split
captions_path, data_path, split, gptmodel, temp, base_dir, job_name = (
args.captions,
args.data,
args.per_vid_captions,
args.gptmodel,
args.temperature,
args.output_dir,
args.job_name,
)
prompt = _PROMPT_TEMPLATE
os.makedirs(base_dir, exist_ok=True)
output_dir = f"{base_dir}/egoschema/{job_name}"
output_dir = os.path.expanduser(output_dir)
os.makedirs(output_dir, exist_ok=True)
save_name = captions_path.rsplit("/", 2)[-1].replace(".jsonl", "")
output_summary_path = f"{output_dir}/{save_name}.jsonl"
print(f"Saving outputs to:{output_summary_path}")
output_summary = open(output_summary_path, "w")
input_summary = [
json.loads(q) for q in open(os.path.expanduser(captions_path), "r")
]
dataset = json.load(open(os.path.expanduser(data_path), "r"))
input_len = len(input_summary)
assert (
input_len % split == 0
), f"input_len%split:{input_len%split}, input_len:{input_len}, split:{split}"
groups = input_len // split
final_prompts = []
final_info = []
for i in tqdm(range(groups)):
sidx = i * split
eidx = (i + 1) * split
desc = ""
timeline = []
for idx, e in enumerate(input_summary[sidx:eidx]):
cur_data = dataset[e["q_uid"]]
desc += e["answer"] + " "
timeline.append(e["timeline"])
if idx == split - 1: # the last of split
action_0 = cur_data["option 0"]
action_1 = cur_data["option 1"]
action_2 = cur_data["option 2"]
action_3 = cur_data["option 3"]
action_4 = cur_data["option 4"]
option_list = ""
option_number_candidate = ["one", "two", "three", "four", "five"]
option_number = option_number_candidate[4]
AllOptNumber = "option 0, option 1, option 2, option 3, option 4"
FocusOptions = ""
alloptions = f"option 0: {action_0}\noption 1: {action_1}\noption 2: {action_2}\noption 3: {action_3}\noption 4: {action_4}"
option_list = f"option 0: {action_0}\noption 1: {action_1}\noption 2: {action_2}\noption 3: {action_3}\noption 4: {action_4}"
FocusOptions += "option 0, option 1, option 2, option 3, option 4"
question = cur_data["question"]
curr_prompt = (
prompt.replace("{Putdesc}", desc)
.replace("{Putquestion}", question)
.replace("{Putoptions}", option_list)
.replace("{PutOptNumber}", option_number)
.replace("{FocusOptions}", FocusOptions)
.replace("{AllOptions}", alloptions)
.replace("{PutAllOptNumber}", AllOptNumber)
)
final_prompts.append(curr_prompt)
CA_option = {}
if "CA" in cur_data:
CA_option = {"CA": cur_data["CA"]}
info = {
"q_uid": e["q_uid"],
"prompt": curr_prompt,
"timeline": timeline,
"question": question,
"option 0": action_0,
"option 1": action_1,
"option 2": action_2,
"option 3": action_3,
"option 4": action_4,
} | CA_option
final_info.append(info)
output_VLM = run_gpt(
texts=final_prompts,
api_keys=list(dict_api.values()),
max_tokens=2000,
model=gptmodel,
temperature=temp,
num_threads=20, # Tune this
backoff_time=1 * 60,
silent=False,
dataset="egoschema",
)
output_VLM = list(output_VLM)
for q_idx, info in enumerate(tqdm(final_info)): # prompt_list = # Q&C
info["answer"] = output_VLM[q_idx]
output_summary.write(json.dumps(info) + "\n")
# finish the summarization for the current question
output_summary.close()
print(f"output_summary_path:{output_summary_path}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--output-dir", type=str)
parser.add_argument("--job-name", type=str, default="run001")
parser.add_argument("--captions", type=str, default="data/ES_captions_gpt4o.jsonl")
parser.add_argument("--data", type=str, default="data/ES_qa_data.json")
parser.add_argument("--per-vid-captions", type=int, default=12)
parser.add_argument("--gptmodel", type=str, default="gpt-3.5-turbo-1106")
parser.add_argument("--temperature", type=float, default=None)
args = parser.parse_args()
eval_model(args)
|