# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 自动阅卷
#
# 初中历史主观题作答数据集

import re

import jieba
import jieba.analyse

import pandas as pd

from functools import reduce

# ## Configure

# ### Re

# Chinese punctuation
re_cn_punc = re.compile(r'[\uff1a\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b:;,.]')

re_cn_score = re.compile(f'\u5404\d+\u5206')

re_slash = re.compile(r'(?!=\d+)-(?!=\d+)')

# question id decode
re_q_decode = re.compile(r'(\d+)(?:\D+(\d+))?')

re_whitespace = re.compile(r'\s+|_x000D_')

re_p_score = re.compile(r"(?:（|\()(?: *)([0-9.]+)分(?: *)(?:）|\))")

# Sub Serial number
re_serial = re.compile(r"(?:[\u2460-\u2468])|(?:（|\()(?: *)([0-9.]+)(?: *)(?:）|\))")

# ### Jieba

stop_word_path = r"cn_stopwords.txt"
stop_words = open(stop_word_path,'r',encoding='utf-8').read().split('\n')+['\n']





# ## Clean Data

data_1 = pd.read_excel('data.xlsx', sheet_name=0)
data_2 = pd.read_excel('data.xlsx', sheet_name=1)
data_3 = pd.read_excel('data.xlsx', sheet_name=2)

question_data = data_1.loc[:, ['题目id（唯一）', '题目文本', '题目分值', '该题标准答案（含得分点）']]

student_data = data_2.loc[:, ['学生id', '题目id', '该生作答文本', '该生作答得分', '该生作答得分点']]

question_data = question_data[~reduce(
    lambda x, y: x[1] | y[1],
    question_data.isna().iteritems()
)]

student_data = student_data[~reduce(
    lambda x, y: x[1] | y[1],
    student_data.isna().iteritems()
)]

question = pd.DataFrame({
    'id': question_data['题目id（唯一）'].apply(
        lambda x: '{0[0]:0>2s}{0[1]:0>2s}'.format([
            item if item else '0'
            for item in re_q_decode.search(x).groups()
        ])
    ).astype('string'),
    'text': question_data['题目文本'].apply(
        lambda x: re_whitespace.sub('', x)
    ).astype('string'),
    'score': question_data['题目分值'].astype(float),
    'answer': question_data['该题标准答案（含得分点）'].apply(
       lambda x: re_whitespace.sub('', x)
    ).astype('string')
})
question = question.reset_index(drop=True)

question.info()

student = pd.DataFrame({
    'id': student_data['学生id'].apply(
        lambda x: int(x) if not pd.isna(x) else '202020000'
    ).astype(int),
    'q_id': student_data['题目id'].apply(
        lambda x: '{0[0]:0>2s}{0[1]:0>2s}'.format([
            item if item else '0'
            for item in re_q_decode.search(x).groups()
        ])
    ).astype('string'),
    'answer': student_data['该生作答文本'].apply(
       lambda x: re_whitespace.sub('', x)
    ).astype('string'),
    'score': student_data['该生作答得分'].astype(float),
    'point': student_data['该生作答得分点'].apply(
       lambda x: re_whitespace.sub('', x)
    ).astype('string')
})
student = student.reset_index(drop=True)

student.info()

question

student['answer'][:100]

student.to_csv('student.csv', index=False, encoding='utf-8_sig')
question.to_csv('question.csv', index=False, encoding='utf-8_sig')


# ## Processing

# ### Answer Points Table

def get_point_score():
    info_dict = {
        "q_id": list(),
        "p_id": list(),
        "point": list(),
        "p_score": list(),
    }
    temp_point_score = []
    
    for k, item in question['answer'].iteritems():
        ans_split = re_p_score.split(item)
        ans_tuple = [
            (ans_split[i*2], float(ans_split[i*2+1])) 
            for i in range(len(ans_split) // 2)
        ]
        if not ans_tuple:
            ans_tuple = [(item, question['score'][k])]
        temp_point_score.append(ans_tuple)
    
    for k, item in enumerate(temp_point_score):
        for i, point in enumerate(item):
            info_dict["q_id"].append(k)
            info_dict["p_id"].append(i)
            info_dict["point"].append(point[0])
            info_dict["p_score"].append(point[1])

    return pd.DataFrame(info_dict)


answer = get_point_score()

answer


# ### Split Point words

# #### Standard Answer

def split_point_words(point):
    text = re_serial.sub("\n", point)
    text = re.split("\n", text)
    text = [
        re_cn_score.sub(
            '', re_slash.sub('', temp)
        )
        for sub_item in text
        for temp in re_cn_punc.split(sub_item)
        if sub_item if temp
    ]
    return text


# +
def cut_point(point):
    cut_list = []
    words_list = split_point_words(point)
    
    for item in words_list:
        cut_list.extend(jieba.lcut(item))

#     cut_list = reduce(
#         lambda x,y: x if y in x else x + [y],
#         [[], ] + sorted(cut_list)
#     )
    
    cut_list = list(filter(
        lambda x: x not in stop_words,
        cut_list
    ))

    return cut_list


# -

def decode_words(point):
    words = cut_point(point)
    return '\t'.join(words)


answer['words'] = answer['point'].apply(
    decode_words
).astype('string')
answer = answer.drop('point', axis=1)

answer

answer.to_csv('answer.csv', index=False, encoding='utf-8_sig')

# #### Student Answer

student['words'] = student['answer'].apply(
    decode_words
).astype('string')

student = student.drop('answer', axis=1)

student

student.to_csv('student.csv', index=False, encoding='utf-8_sig')

# ### IDF

# +
# jieba.analyse.extract_tags?
# -

jieba.analyse.extract_tags(
    "第一次国共合作的形成加快了中国革命前进步伐.合作失败后，使党开始成长，懂得了进行土地革命",
    withWeight=True
)





StarcoderdataJupyterScriptsDedupFiltered