# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: recsys_deepconv
#     language: python
#     name: recsys_deepconv
# ---

# %matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os


def get_word_counts(dataset):
    data_dir=f'~/data/twconv/{dataset}'
    data_dir=os.path.expanduser(data_dir)
    fpath = os.path.join(data_dir, 'alldata', 'train.csv')
    print(fpath)
    train = pd.read_csv(fpath)
    train['source_wcount'] = train.source.apply(lambda x: len(x.split(' ')))
    train['target_wcount'] = train.target.apply(lambda x: len(x.split(' ')))
    plt.figure(figsize=(20,8))
    plt.subplot(231)
    train['source_wcount'].plot.hist(bins=50)
    plt.yscale('log')
    plt.subplot(232)
    train['target_wcount'].plot.hist(bins=50)
    plt.yscale('log')


get_word_counts('twconv_2011_trec')

get_word_counts('twconv_2016_usersec')



StarcoderdataJupyterScriptsDedupFiltered