# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
import os, sys
cwd = os.getcwd()
project_path = cwd[:cwd.find('pygents')+7]
if project_path not in sys.path: sys.path.append(project_path)
os.chdir(project_path) 

from importlib import reload  # Python 3.4+

import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import pickle

#force reimport
if 'pygents.util' in sys.modules:
    del sys.modules['pygents.util']
if 'pygents.text' in sys.modules:
    del sys.modules['pygents.text']
if 'pygents.plot' in sys.modules:
    del sys.modules['pygents.plot']
if 'pygents.token' in sys.modules:
    del sys.modules['pygents.token']

from pygents.util import * 
from pygents.text import * 
from pygents.plot import * 
from pygents.token import * 


# +
# https://www.vengaglobal.com/blog/simplified-traditional-chinese-mandarin-cantonese/

# Target Market  Written      Spoken
# -------------------------------------
# China          Simplified   Mandarin
# Singapore      Simplified   Mandarin
# Taiwan         Traditional  Mandarin
# Hong Kong      Traditional  Cantonese

# Lexicon:
# http://www.chineselexicaldatabase.com/download.php - used below
# <NAME>., <NAME>., <NAME>. & <NAME>. (2018). Chinese Lexical Database (CLD): A large-scale lexical database for simplified Mandarin Chinese. Behavior Research Methods, https://doi.org/10.3758/s13428-018-1038-3.
# 48644 words
# http://crr.ugent.be/programs-data/subtitle-frequencies/subtlex-ch
# 99121 words
# https://www.plecoforums.com/threads/word-frequency-list-based-on-a-15-billion-character-corpus-bcc-blcu-chinese-corpus.5859/
# 1048575 words

# Corpora:
# https://www.openslr.org/38/ - test-audio corpus, not relevant
# https://github.com/CLUEbenchmark/CLUECorpus2020/ - email request sent
# https://github.com/brightmart/nlp_chinese_corpus - nearly same as above downloaded, used further

# TODO:
# https://metatext.io/datasets/nlp-chinese-corpus - paper with word segmentation

# -

path = '../../nlp/corpora/Chinese/'


# ## Lexicon

cld_df = pd.read_csv(os.path.join(path,'lexicon/chineselexicaldatabase2.1.txt'))
len(cld_df)

cld_df

cld_df[['Word']]


cld_df = pd.read_csv(os.path.join(path,'lexicon/bcc_global_wordfreq.release_UTF-8.txt'),sep='\t')
print(len(cld_df))
cld_df

cld_df = pd.read_csv(os.path.join(path,'lexicon/SUBTLEX-CH-WF.txt'),sep='\t')
print(len(cld_df))
cld_df








# ## Corpora

#out-of-memory
#news2016zh_valid_df = pd.read_json('data/corpora/chinese/new2016zh/news2016zh_train.json',encoding = 'utf-8-sig', lines=True)
news2016zh_valid_df = pd.read_json(os.path.join(path,'clue/new2016zh/news2016zh_valid.json'),encoding = 'utf-8-sig', lines=True)
len(news2016zh_valid_df)


news2016zh_valid_df


def zh_clue_json2text(path,filename):
    with open(os.path.join(path,filename+'.json')) as file:
        with open(os.path.join(path,filename+'.txt'), 'w') as fout:
            while True:
                line = file.readline()
                if not line:
                    break
                j = json.loads(line)
                #print('title',j['title'])
                #print('desc',j['desc'])
                #print('content',j['content'])
                fout.write(j['title'])
                fout.write('\n')    
                fout.write(j['desc'])
                fout.write('\n')    
                fout.write(j['content'])
                fout.write('\n')    


#do this once!
#zh_clue_json2text(path,'clue/new2016zh/news2016zh_valid')


#do this once!
#zh_clue_json2text(path,'clue/new2016zh/news2016zh_train')


#check if context is present
n_counters1 = context_save_load(None,'chinese_news2016zh_valid',folder='data/models/')
len(n_counters1)

if n_counters1 is None or len(n_counters1) < 1:
    max_n = 3 # in case of Chinese!?
    n_counters1 = grams_init(max_n)
    cnt = 0
    with open(os.path.join(path, 'clue/new2016zh/news2016zh_valid.txt'),errors='ignore') as f:
        while True:
            line = f.readline()
            if not line:
                break
            cnt += 1
            if (cnt % 1000) == 0:
                print(cnt,line)
            text = preprocess_text(line)
            text_grams_count(n_counters1,text,max_n)
    print(cnt)
    context_save_load(n_counters1,'chinese_news2016zh_valid',folder='data/models/')


dfs = []
for i in range(len(n_counters1)):
    counter = n_counters1[i]
    df = pd.DataFrame([(gram, counter[gram]) for gram in counter],columns=['gram','freq'])
    df['log'] = np.log10(df['freq'])
    df.sort_values('freq',ascending=False,inplace=True)
    df.title = str(1+i)
    dfs.append(df)

dfs[0][:20][['gram','freq']]
# 的 - of
# 是 - yes
# 在 - exist
# 不 - do not

dfs[1][:20][['gram','freq']]
# 一个 - one
# 公司 - company
# 中国 - china
# 我们 - us/ourselves
# 可以 - can

dfs[2][:20][['gram','freq']]
# 自己的 - my own
# ，我们 - , us
# 互联网 - the internet

# +
#https://chowdera.com/2022/03/202203280859161240.html

#http://anqin007.blogspot.com/2018/12/show-chinese-characters-in-matplotlib.html
from pylab import mpl
mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['axes.unicode_minus'] = False

plt.rcParams["figure.figsize"] = (20,20)
for df in dfs:
    p = df[:100][['gram','freq']].plot.barh(x='gram'); p.invert_yaxis();
    p.set_title(df.title,fontsize = 32)
    plt.show()
# -

plt.rcParams["figure.figsize"] = (20,20)
for df in dfs:
    p = df[:100][['gram','log']].plot.barh(x='gram'); p.invert_yaxis();
    p.set_title(df.title,fontsize = 32)
    plt.show()







StarcoderdataJupyterScriptsDedupFiltered