# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

import os
import numpy as np

no_convert = ['<s>', '</s>', '<eof>', '<unk>']

train = ''
with open('../data/ptb-punct/train.txt') as f:
    for i, l in enumerate(f):
#         print("\n" in l) 
        for word in l.split():
            if word in no_convert:
                train+=word
                train+=" "
            else:
                for c in word:
                    train+=c
                    train+=" "
            train+= "<space> "
        train+="\n"
#         if i>5:
#             break
# print train

with open('../data/ptb-punct/train_char.txt', 'w') as f:
    f.write(train)

# +
valid = ''
with open('../data/ptb-punct/train.txt') as f:
    for i, l in enumerate(f):
#         print("\n" in l) 
        for word in l.split():
            if word in no_convert:
                valid+=word
                valid+=" "
            else:
                for c in word:
                    valid+=c
                    train+=" "
            train+= "<space> "
        train+="\n"


# -

with open('../data/ptb-punct/valid_char.txt', 'w') as f:
    f.write(valid)



StarcoderdataJupyterScriptsDedupFiltered