# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + uuid="fb1a49fb-2775-43ed-ba80-cd439465770d"
import pandas as pd
from nltk.translate.bleu_score import sentence_bleu

references = ["吴承恩是著名文学家"] * 6
candidates = """
吴承恩是著名文学家
吴承恩是大作家
吴承恩是作曲家
吴承恩是运动员
吴承恩不是著名文学家
风马牛不相及
""".strip().split("\n")

bleu_score = []
for ref, cand in zip(references, candidates):
    bleu_score.append(sentence_bleu([ref], cand))
bleu_score = pd.Series(bleu_score)
    
print(bleu_score)
bleurt_score = pd.Series([0.8896132111549377, 0.22806376218795776, 0.22379213571548462, 0.21101689338684082, 0.5763261318206787, -0.007796883583068848])
bleu_score - bleurt_score

StarcoderdataJupyterScriptsDedupFiltered