# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda root]
#     language: python
#     name: conda-root-py
# ---

import pickle
from collections import Counter
import codecs


def read_sentences(filepath):
    sentences = []
    with codecs.open(filepath, encoding="utf-8", mode="r") as fp:
        for sentence in fp:
            sentences.append(sentence.lower())
    return sentences


def read_sentences_format2(filepath):
    X = []
    Y = []
    with codecs.open(filepath, encoding = "utf-8", mode = "r") as fp:
        for sentence in fp:
            splits = sentence.split("\t")
            X.append(splits[3].strip().lower())
            Y.append(splits[4].strip().lower())
    return X, Y


def create_dataset(l1_sentences, l2_sentences):
    l1_vocab_dict = Counter(word.strip(',." ;:)(][?!-\'') for sentence in l1_sentences for word in sentence.split())
    l2_vocab_dict = Counter(word.strip(',." ;:)(|][?!<>a-zA-Z') for sentence in l2_sentences for word in sentence.split())
    #l1_vocab_dict = Counter(word for sentence in l1_sentences for word in sentence.split())
    #l2_vocab_dict = Counter(word for sentence in l2_sentences for word in sentence.split())
    
    l1_vocab = list(map(lambda x: x[0], sorted(l1_vocab_dict.items(), key = lambda x: -x[1])))
    l2_vocab = list(map(lambda x: x[0], sorted(l2_vocab_dict.items(), key = lambda x: -x[1])))
    
    # Limit the vocabulary size. Consider only the top 20,000 and 30,000 words respectively
    l1_vocab = l1_vocab[:30000]
    l2_vocab = l2_vocab[:30000]
    
    # Build a Word to Index Dictionary for English
    start_idx = 2
    l1_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(l1_vocab)])
    l1_word2idx['<ukn>'] = 0 # Unknown words
    l1_word2idx['<pad>'] = 1 # Padding word
    
    # Build an Index to Word Dictionary for English using the already created Word to Index Dictionary
    l1_idx2word = dict([(idx, word) for word, idx in l1_word2idx.items()])
    
    # Build a Word to Index Dictionary for Hindi
    start_idx = 4
    l2_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(l2_vocab)])
    l2_word2idx['<ukn>'] = 0 # Unknown
    l2_word2idx['<go>']  = 1 
    l2_word2idx['<eos>'] = 2 # End of sentence
    l2_word2idx['<pad>'] = 3 # Padding
    
    # Build an Index to Word Dictionary for Hindi using the already created Word to Index Dictionary
    l2_idx2word = dict([(idx, word) for word, idx in l2_word2idx.items()])

    # Encode words in senteces by their index in Vocabulary
    x = [[l1_word2idx.get(word.strip(',." ;:)(][?!-\''), 0) for word in sentence.split()] for sentence in l1_sentences]
    y = [[l2_word2idx.get(word.strip(',." ;:)(|][?!<>a-zA-Z'), 0) for word in sentence.split()] for sentence in l2_sentences]
    #x = [[l1_word2idx.get(word, 0) for word in sentence.split()] for sentence in l1_sentences]
    #y = [[l2_word2idx.get(word, 0) for word in sentence.split()] for sentence in l2_sentences]
    
    X = []
    Y = []
    for i in range(len(x)):
        n1 = len(x[i])
        n2 = len(y[i])
        n = n1 if n1 < n2 else n2
        if abs(n1 - n2) < 0.3 * n:
            if n1 <= 20 and n2 <= 20:
                X.append(x[i])
                Y.append(y[i])
    
    return X, Y, l1_word2idx, l1_idx2word, l1_vocab, l2_word2idx, l2_idx2word, l2_vocab


def save_dataset(filepath, obj):
    with open(filepath, 'wb') as fp:
        pickle.dump(obj, fp, -1)


def read_dataset(filepath):
    with open(filepath, 'rb') as fp:
        return pickle.load(fp)


# +
def main():
    dataset_save_location = "D:/NLP Project/Hindi English/data.p"
    
    X, Y = read_sentences_format2("D:/NLP Project/Hindi English/hindencorp05.plaintext")
    out_obj = {'X':X, 'Y':Y}
    pickle.dump(out_obj, open("D:/NLP Project/Hindi English/hindencorp05.p", "wb"))
    #en_sentences = read_sentences(en_path)
    #hi_sentences = read_sentences(hi_path)
    #en_sentences.extend(X)
    #hi_sentences.extend(Y)
    en_sentences = X
    hi_sentences = Y
    #save_dataset(dataset_save_location , create_dataset(en_sentences, hi_sentences))

if __name__ == '__main__':
    main()
# -

X, Y = read_sentences_format2("D:/NLP Project/Hindi English/hindencorp05.plaintext")
print(X[:2])
print(Y[:2])

StarcoderdataJupyterScriptsDedupFiltered