<jupyter_start><jupyter_text>https://www.cs.cornell.edu/projects/kddcup/datasets.html<jupyter_code>!tar xzvf hep-th-2003.tar.gz
!ls -hal 2003/ | wc -l<jupyter_output><empty_output><jupyter_text>load libraries<jupyter_code># https://github.com/alvinwan/tex2py
!pip install tex2py
# https://github.com/alvinwan/TexSoup
!pip install texsoup
import re
import time
import glob
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>load data<jupyter_code>list_of_files = glob.glob('hep-th/2003/*')
len(list_of_files)<jupyter_output><empty_output><jupyter_text>find relevant tags in latex<jupyter_code>start_time=time.time()
reslts={}
number_of_eq=0
for this_file in list_of_files:
    with open(this_file,'rb') as f:
        data = f.read()
    
    resp = re.findall('\\\\begin{(?:eqnarray|equation|multiline)}.*?end{(?:eqnarray|equation|multiline)}',str(data),re.DOTALL)
#    print(this_file)
    for eq in resp:
        number_of_eq+=1
        try:
            reslts[this_file].append(eq)
        except KeyError:
            reslts[this_file]=[]
            reslts[this_file].append(eq)
#        print('  ',eq,'\n')

print(round(time.time()-start_time,2),'seconds')
number_of_eq
len(reslts.keys())
reslts[list(reslts.keys())[0]]
reslts[list(reslts.keys())[1]][0]<jupyter_output><empty_output><jupyter_text>visualize results<jupyter_code>number_of_eq_per_file=[]
for k,v in reslts.items():
    number_of_eq_per_file.append(len(v))
_=plt.hist(number_of_eq_per_file,bins=100)
_=plt.ylabel('number of expressions')
_=plt.xlabel('number of documents')
_=plt.hist(number_of_eq_per_file,bins=100)
_=plt.yscale('log', nonposy='clip')
_=plt.ylabel('number of expressions')
_=plt.xlabel('number of documents')<jupyter_output><empty_output><jupyter_text>parse the Latexoptions: * http://plastex.sourceforge.net/plastex/sect0025.html* tex2py* texsoup<jupyter_code># https://github.com/alvinwan/tex2py
from tex2py import tex2py
list_of_files[1]
with open(list_of_files[1]) as f: data = f.read()
toc = tex2py(data)<jupyter_output><empty_output><jupyter_text>toc.valid_tags<jupyter_code>from TexSoup import TexSoup
soup = TexSoup(data)
soup.equation
lst = list(soup.find_all('equation'))
len(lst) # how many equations are in the document?
lst[0] # show the first match
lst[0][0] # what is the string inside the "begin{equation}"?
lst[1]
lst[1][0]
for this_eq in soup.find_all('equation'):
    print(this_eq[0])<jupyter_output>\label{planewave}
\begin{split}
ds^2 & = 2dx^+dx^--\m^2\vec{x}^2\bigl(dx^+\bigr)^2+d\vec{x}^2\,,\\
F_5 & = 4\m dx^+\wedge\bigl(dx^1\wedge dx^2\wedge dx^3\wedge dx^4+dx^5\wedge dx^6\wedge dx^7\wedge dx^8\bigr)\,.


\o_n=\sqrt{n^2+\bigl(\m\a'p^+\bigr)^2}\,,\qquad n\in\Nop\,,

\label{dict}
\frac{1}{\m}H = \D-J\,,\qquad \frac{1}{\bigl(\m\a'p^+\bigr)^2} = \frac{g^2_{\text{YM}}N}{J^2}\equiv \l'\,,\qquad
4\pi g_{\text{s}}\bigl(\m\a'p^+\bigr)^2 = \frac{J^2}{N}\equiv g_2


N\to\infty\quad\text{and}\quad J\to\infty\quad\text{with}\quad\frac{J^2}{N}\quad
\text{fixed}\,,\quad g_{\text{YM}}\quad\text{fixed}\,,


S_{\mbox{\scriptsize bos.}(r)}=\frac{e(\a_r)}{4\pi\a'}\int\,d\t\int_0^{2\pi|\a_r|}
\,d\s_r\bigl[\dot{x}_r^2-x^{\prime\,2}_r-\m^2x_r^2\bigr]\,,


\begin{split}
x_r^I(\s_r)& = x_{0(r)}^I+\sqrt{2}\sum_{n=1}^{\infty}
\bigl(x_{n(r)}^I\cos\frac{n\s_r}{|\a_r|}+x_{-n(r)}^I\sin\frac{n\s_r}{|\a_r|}\bigr)\,,\\
p_r^I(\s_r) & =\frac{1}{2\pi|\a_r|}\bigl[p_{0(r)}^I+\sqrt{2}\sum_{n=1}^{\infty}
\bigl(p_{n(r[...]
StarcoderdataJupyterStructuredCleanDedup