<jupyter_start><jupyter_text>Use regex to process the feedbackThis is the baseline for self-feeding chatbot<jupyter_code>import re
import os
import pandas as pd
from google.colab import drive
drive.mount('/gdrive')
DATASET='/gdrive/My Drive/ParlAI/data/self_feeding'
os.chdir(DATASET)
os.listdir()<jupyter_output><empty_output><jupyter_text><jupyter_code>def processed_feedback(file_name, file_dir):
    file_path = os.path.join(file_dir, file_name)
    df = pd.read_json(file_path, lines=True)
    response_list = df['response'].to_list()

    processed_responses = []

    for idx, sentence in enumerate(response_list):

        match_choice = re.search('yes or no', sentence)
        if match_choice:
            processed_responses.append("yes")
            continue

        match_greeting = re.search('greeting|hi |hello', sentence)
        if match_greeting:
            processed_responses.append("hello, how are you?")
            continue

        match_job = re.search('job|career|living', sentence)
        if match_job:
            processed_responses.append("I am a teacher, what about you?")
            continue

        match_mess = re.search('messed up',sentence)
        if match_mess:
            processed_responses.append("Sorry, I am not good at this topic. Do you want to talk about anything else?")
            continue

        match_country = re.search('country',sentence)
        if match_mess:
            processed_responses.append("I am a robot, I don't live in any country")
            continue

        match_filler = re.search('you could|you should|said|saying|say|tell|told|admit|ask|answer|talk|yes|no', sentence)

        if match_filler: 
            # processed_sentence = sentence          
            processed_sentence = re.sub("you could have|you should have|you could|you should", '', sentence).strip()
            processed_sentence = re.sub("^.*said|^.*saying|^.*say|^.*tell |^.*told |^.*admit |^.*asked |^.*ask |^.*answer |^.*answered |^.*talked |^.*talk ", '', processed_sentence).strip()
            processed_sentence = re.sub("^.*about|^me |^of ", '', processed_sentence).strip()

            piceses = processed_sentence.split(' or ')
            processed_sentence = piceses[0].strip()
            for piece in piceses:
                piece = piece.strip()
                if len(piece) > len(processed_sentence):
                    processed_sentence = piece
            # remove                
            processed_sentence = re.sub("^if |^whether |^not ", '', processed_sentence).strip()

            # replace subject
            processed_sentence = re.sub("you ", 'i ', processed_sentence).strip()
            processed_sentence = re.sub("you are |i are ", 'i am ', processed_sentence).strip()
            processed_sentence = re.sub("your ", 'my ', processed_sentence).strip()        
            processed_sentence = re.sub("you\'ve ", 'i\'ve ', processed_sentence).strip()
            processed_sentence = re.sub("you were | i were", 'i was', processed_sentence).strip()            
            processed_sentence = re.sub("you're ", 'i\'m ', processed_sentence).strip()

            
            # Remove starting space and comma
            processed_sentence = re.sub("\“|\”", '', processed_sentence).strip() 
            processed_sentence = processed_sentence.lstrip(':|,|\"|\'|-|.| ')
            processed_sentence = processed_sentence.rstrip('\"|\'| ')

            if len(processed_sentence) > 0:
                processed_responses.append(processed_sentence)
            else:
                # print(processed_sentence,"<<<<<<", sentence)
                processed_responses.append(sentence)

            # print(processed_sentence,"<<<<<<", sentence)
            s = processed_sentence + "<<<<<<" + sentence +'\n'
            fd_out.writelines(s)

        else:
            processed_responses.append(sentence)

    df['response'] = processed_responses
    file_name='regex_'+file_name
    print(file_name)
    df.to_json(file_name, orient='records',lines=True)
        
# Process all feedback files
# file_list = ['train_fb.txt', 'train_fb_a.txt', 'train_fb_b.txt', 'test_fb.txt', 'valid_fb.txt']
# file_dir = os.path.abspath(os.path.join(os.getcwd(), '..', 'self_feeding_bak'))

file_list = ['train_fb.txt']
file_dir = os.getcwd()


with open('processed_feedback.txt', 'w') as fd_out:
    for file_name in file_list:
        processed_feedback(file_name, file_dir)
!wc -l train_fb.txt
!wc -l processed_feedback.txt
df = pd.read_json("train_fb.txt", lines=True)
response_list = df['response'].to_list()
# with open("regex_test.txt", 'r') as f_in, open('processed_feedback.txt', 'w') as fd_out:
#     lines = f_in.readlines()
#     processed_responses = []
#     for sentence in lines:
#         sentence = sentence.strip()

#         match_choice = re.search('yes or no', sentence)
#         if match_choice:
#             processed_responses.append("yes")
#             continue

#         match_greeting = re.search('greeting|hi |hello', sentence)
#         if match_greeting:
#             processed_responses.append("Hello, how are you?")
#             continue

#         match_job = re.search('job|career|living', sentence)
#         if match_job:
#             processed_responses.append("I am a teacher, what about you?")
#             continue

#         match_mess = re.search('messed up',sentence)
#         if match_mess:
#             processed_responses.append("Sorry, I am not good at this topic. Do you want to talk about anything else?")
#             continue

#         match_country = re.search('country',sentence)
#         if match_mess:
#             processed_responses.append("I am a robot, I don't live in any country")
#             continue

#         match_filler = re.search('you could|you should|said|saying|say|tell|told|admit|ask|answer|talk|yes|no', sentence)

#         if match_filler: 
#             # processed_sentence = sentence          
#             processed_sentence = re.sub("you could have|you should have|you could|you should", '', sentence).strip()
#             processed_sentence = re.sub("^.*said|^.*saying|^.*say|^.*tell |^.*told |^.*admit |^.*asked |^.*ask |^.*answer |^.*answered |^.*talked |^.*talk |^.*that ", '', processed_sentence).strip()
#             processed_sentence = re.sub("^.*about|^me|that", '', processed_sentence).strip()

#             piceses = processed_sentence.split(' or ')
#             processed_sentence = piceses[0].strip()
#             for piece in piceses:
#                 piece = piece.strip()
#                 if len(piece) > len(processed_sentence):
#                     processed_sentence = piece
#             # remove                
#             processed_sentence = re.sub("^if|^whether|^not", '', processed_sentence).strip()

#             # replace subject
#             processed_sentence = re.sub("you are ", 'i am ', processed_sentence).strip()
#             processed_sentence = re.sub("your ", 'my ', processed_sentence).strip()        
#             processed_sentence = re.sub("you\'ve ", 'i\'ve ', processed_sentence).strip()
#             processed_sentence = re.sub("you were", 'i was', processed_sentence).strip()
#             processed_sentence = re.sub("you ", 'i ', processed_sentence).strip()
#             processed_sentence = re.sub("you're ", 'i\'m ', processed_sentence).strip()

            
#             # Remove starting space and comma
#             processed_sentence = re.sub("\“|\”", '', processed_sentence).strip() 
#             processed_sentence = processed_sentence.lstrip(':|,|\"|\'|-|.| ')
#             processed_sentence = processed_sentence.rstrip('\"|\'| ')

#             if len(processed_sentence) > 0:
#                 processed_responses.append(processed_sentence)
#             else:
#                 processed_responses.append(sentence)

#         else:
#             processed_responses.append(sentence)
             
#     for res in processed_responses:
#         fd_out.write(res+'\n')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup