precompress_run.sh

# python3 offline_compress_m1.py \
#     --input_dir data/m1 \
#     --output_dir test_data/m1 \
#     --model_path /mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts/m1_checkpoints/m1_6M_lr1e-2_steps50k_bs128_seqlen512/checkpoints/0000050000

total_jsonls=8
total_jobs=2

# --firstbyte_prob_path /mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts/ac_unigram_probs/python500k_unigram_prob.json \
for JSONL_IDX in $(seq 1 $total_jsonls); do
    for index in $(seq 0 $((total_jobs - 1))); do
        echo "Starting job $index..."

        GPU_IDX=$(( JSONL_IDX - 1 ))
        CUDA_VISIBLE_DEVICES=${GPU_IDX} python3 offline_compress_m1_outputwindow.py \
            --input_file /mnt/hdfs/user/linzheng/data/ocpython_subsampled_50G/ocp.chunk.${JSONL_IDX}.jsonl \
            --output_dir ocpython_subsampled_50G_outputwindow_16 \
            --model_path /mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts/m1_checkpoints/m1_6M_lr1e-2_steps50k_bs128_seqlen512/checkpoints/0000050000 \
            --batch_size 1024 --chunk_size 1024 \
            --process_id $index --num_processes $total_jobs \
            --output_window_size 16 > gpu${GPU_IDX}_process${index}_total${total_jobs}.log 2>&1 &
    done
done