# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/catmium/bkkdreams/blob/master/extract_budget63.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="Hlg7CCRFyMDq" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 305} outputId="bb7691eb-645d-45df-97fe-4fff1693f291"
# !pip install kora -q
# !wget http://www-us.apache.org/dist/pdfbox/2.0.21/pdfbox-app-2.0.21.jar -O pdfbox.jar

# + id="adyWULae3Ect" colab_type="code" colab={}
from kora.bs import Soup
from google.colab import files
import re
import pandas as pd

# + id="8MnbxoLw3Hhu" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 343} outputId="6f86654d-94f4-4c0c-d59e-589432567674"
# Download BKK Budget B.E. 63 PDF file
# !wget -O budget63.pdf https://drive.google.com/uc?id=1VKlLclTm7Dwwwe4ttK0u8dzvjopQZwPj

# + id="d1u8A5R23Mth" colab_type="code" colab={}
# PDF -> HTML
# !java -jar pdfbox.jar ExtractText -html budget63.pdf budget63.html

# + id="f6n9onid0FDx" colab_type="code" colab={}
# https://gist.github.com/anonymous/11019e4cba687bd846db
import html
import re

pua = {
    '63233': '&#3636;',
    '63234': '&#3637;',
    '63235': '&#3638;',
    '63236': '&#3639;',
    '63237': '&#3656;',
    '63238': '&#3657;',
    '63242': '&#3656;',
    '63243': '&#3657;',
    '63246': '&#3660;',
    '63248': '&#3633;',
    '63250': '&#3655;',
    '63251': '&#3656;',
    '63252': '&#3657;'
}

def thaiPUA(matchobj):
    return pua[matchobj.group(1)]

p = re.compile(r'\&\#(\d{5,})\;')

outputf = open('budget63_new.html', 'w')
inputf = open('budget63.html', 'r')
for line in inputf:
    text = p.sub(thaiPUA, line)
    outputf.writelines(html.unescape(text))
inputf.close()
outputf.close()

# + id="KXZaV1yz0vff" colab_type="code" colab={}
# https://gist.github.com/anonymous/ac7269b39773d78a24a1

import re
from html.parser import HTMLParser

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.strict = False
        self.convert_charrefs= True
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

digits = {
    '๐': '0',
    '๑': '1',
    '๒': '2',
    '๓': '3',
    '๔': '4',
    '๕': '5',
    '๖': '6',
    '๗': '7',
    '๘': '8',
    '๙': '9'
}

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

def thaiDigits(matchobj):
    return digits[matchobj.group(0)]

def tonemarkPos(matchobj):
    return matchobj.group(2) + matchobj.group(1)

def tonemarkSpace(matchobj):
    return matchobj.group(1) + matchobj.group(2)

def article(matchobj):
    num = matchobj.group(1)
    return '\nมาตรา ' + num + matchobj.group(2)

p1 = re.compile(r'[๐๑๒๓๔๕๖๗๘๙]')
p2 = re.compile(r'([่้๊๋])([ัุูิีึื])')
p3 = re.compile(r'([ัุูิีึื])\s+([่้๊๋])')
p4 = re.compile(r'[ ]+$', re.MULTILINE)
# p5 = re.compile(r'[0-9]+\n+ร่างเบื้องต้น วันศุกร์ ที่ 29 มกราคม 2559\n+')
# p6 = re.compile(r'^\s+มาตรา\s+([0-9]+)(.*?)(?=(\n\s+มาตรา\s+[0-9]+))', re.DOTALL|re.MULTILINE)
p7 = re.compile(r'\s+(?=[ัุูิีึื])')

inputf = open('budget63_new.html', 'r')
text = inputf.read()
inputf.close()

# text = strip_tags(text)
text = p1.sub(thaiDigits, text)
text = re.compile(r'(?<=\d),(?=\d)').sub('', text)
text = p2.sub(tonemarkPos, text)
text = p3.sub(tonemarkSpace, text)
text = p4.sub('', text)
# text = p5.sub('', text)
# text = p6.sub(article, text)
text = p7.sub('', text)
text = re.compile(r'(?<=[ก-ฮ])\sา').sub('ำ', text)
text = re.compile(r'(?<=น)\s{1,2}า').sub('ำ', text)
text = re.compile(r'(?<=น้)\s{1,2}า').sub('ำ', text)
text = re.compile(r'(?<=ระบาย)นำ').sub('น้ำ', text)
text = re.compile(r'(?<=คุณภาพ)นำ').sub('น้ำ', text)


outputf = open('budget63_new_new.html', 'w')
outputf.write(text)
outputf.close()

# + id="EtmP58xA3VU7" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="ce8ac680-d039-436f-a8c7-f92fac901e85"
divs = Soup('budget63_new_new.html').select("body > div")

pages = divs[2:]
page = pages[2]
paras = page.select('p')

print(len(divs),len(paras))

# + id="q9Xr4BGgCtiL" colab_type="code" colab={}


# + id="eGdiPzwjE6i4" colab_type="code" colab={}
def gen_para(start=2, end=None):
  if end:
    for page in pages[start:end]:
      yield from page.select('p')
  else:
    for page in pages[start:]:
      yield from page.select('p')

def gen_top_lvl():
  for p in gen_para():
    if p.contents[0].name=='b':
      yield p


# + id="Rv6-LDXEUa7V" colab_type="code" colab={}
budget_top_types = list()
budget_top_amounts = list()

for p in gen_top_lvl():
  line = p.text.split('\n')[0]
  if line.startswith('('):
    type_ = re.findall(r'(?<=\) ).*(?= รวม)', line)[0]
    amt_ = float(re.findall(r'(?<= รวม )\d+(?= บาท)', line)[0])
    budget_top_types.append(type_)
    budget_top_amounts.append(amt_)
budget_top_lv_df = pd.DataFrame({'budget_top_lvl_type': budget_top_types, 'amount': budget_top_amounts} )

# + id="stvEyeElUwWi" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 204} outputId="b206df57-0912-443b-e627-8928d90b5894"
budget_top_lv_df.head()

# + id="G5ZzW9dhol0J" colab_type="code" colab={}


# + id="RhFrfvdTo4i5" colab_type="code" colab={}
from bokeh.plotting import figure, show
from bokeh.io import output_notebook
from bokeh.models import LabelSet, ColumnDataSource
from math import pi

# + id="gvcvjGfFq1gi" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 917} outputId="b64684a3-97a4-4934-b24d-83c9d438a71b"
budget_top_lv_df = budget_top_lv_df.sort_values('amount', ascending=False)

output_notebook()
bplot = figure(x_range=budget_top_lv_df['budget_top_lvl_type'], height=900, width=1500, title="Budget",
           toolbar_location=None, tools="")
bplot.vbar(x=budget_top_lv_df['budget_top_lvl_type'], top=budget_top_lv_df['amount'], width=0.9)
bplot.xaxis.major_label_orientation = pi/4

show(bplot)

# + id="9v83Dv1RrBnO" colab_type="code" colab={}
res = list()
for p in gen_para(2,3):
  if p.contents[0].name=='b':
    line = p.text.split('\n')[0]
    if line.startswith('('):
      org = re.findall(r'(?<=\) ).*(?= รวม)', line)[0]
  for ln in p.text.split('\n'):
    budget_type = 'งบกลาง'
    if re.match(r'^(\d+\.\s)(.*)(บาท)', ln) != None:
      pw = re.compile(r'^(\d+\.\s)').sub('', ln)
      pw_type = re.match(r'(.*) (\d+)', pw).group(1)
      pw_amt = float(re.match(r'(.*) (\d+)', pw).group(2))
      res.append((org, pw_type, budget_type, pw_amt))
for p in gen_para(3):
  if p.contents[0].name=='b':
    line = p.text.split('\n')[0]
    if line.startswith('('):
      org = re.findall(r'(?<=\) ).*(?= รวม)', line)[0]
  for ln in p.text.split('\n'):
    if re.match(r'^(รายจ่าย)(ประจำ|พิเศษ)', ln) != None:
      budget_type = re.match(r'^(รายจ่าย)(ประจำ|พิเศษ)', ln).group(2)
    if re.match(r'(^[ก-ฮ]{1}\.\s)?(แผนงาน|งบ)(.*)(บาท)', ln) != None:
      pw = re.compile(r'(^[ก-ฮ]{1}\.\s)').sub('', ln)
      pw_type = re.match(r'(.*) (\d+)', pw).group(1)
      pw_amt = float(re.match(r'(.*) (\d+)', pw).group(2))
      res.append((org, pw_type, budget_type, pw_amt))

# + id="HxbnqFHoDJoI" colab_type="code" colab={}
odba_df = pd.DataFrame(res, columns=['org', 'dep', 'budget_type', 'amount'])

# + id="WgQ1nC_eQPDW" colab_type="code" colab={}
odba_g_dep = odba_df[odba_df['org'] != 'งบกลาง'].groupby('dep').agg({'amount': 'sum'}).reset_index()

# + id="PvW7FkArVy-5" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="958a139c-ee3c-42d4-fdf8-1ecf124218dc"
odba_g_dep = odba_g_dep.sort_values('amount', ascending=False)
source = ColumnDataSource(odba_g_dep)
output_notebook()
bplot = figure(x_range=source.data['dep'], height=1000, width=1200, title="Budget by Dep",
           toolbar_location=None, tools="")
bplot.vbar(x='dep', 
           top='amount', 
           width=0.85, source=source)
bplot.xaxis.major_label_orientation = pi/4
# labels = LabelSet(x='x', y='y', text='y', level='glyph',
#         x_offset=-13.5, y_offset=0, source=source, render_mode='canvas')

# bplot.add_layout(LabelSet(x='dep', y='amount', text='amount',level='glyph', source=source, render_mode='canvas', angle=pi/14, text_font_size='10px'))
show(bplot)

# + id="Og7AA5UAZc69" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 717} outputId="64de466f-1c08-4fa1-a60f-f0f4ca290e12"
from bokeh.io import output_file, show
from bokeh.palettes import Category20c, Greys
from bokeh.plotting import figure
from bokeh.transform import cumsum

output_file("pie.html")


odba_g_dep['angle'] = odba_g_dep['amount']/odba_g_dep['amount'].sum() * 2*pi
odba_g_dep['pc'] = round(odba_g_dep['amount']/odba_g_dep['amount'].sum() * 100)
try:
  odba_g_dep['color'] = Category20c[min(odba_g_dep.shape[0],20)] + Greys[-1*(20-odba_g_dep.shape[0])]
except:
  dba_g_dep['color'] = Category20c[min(odba_g_dep.shape[0],20)]

p = figure(plot_height=700, plot_width=1000, title="Pie Chart", toolbar_location=None,
           tools="hover", tooltips="@dep: @amount, @pc%", x_range=(-0.5, 1.0))

p.wedge(x=0, y=1, radius=0.4,
        start_angle=cumsum('angle', include_zero=True), end_angle=cumsum('angle'),
        line_color="white", fill_color='color', legend_field='dep', source=odba_g_dep)

p.axis.axis_label=None
p.axis.visible=False
p.grid.grid_line_color = None

show(p)

# + id="B90tiX7cg56N" colab_type="code" colab={}
odba_df.to_csv('bma_budget63_extracted.csv', index=False)

StarcoderdataJupyterScriptsDedupFiltered