# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: base
#     language: python
#     name: base
# ---

import pandas as pd
import numpy as np
import glob
import warnings
from bs4 import BeautifulSoup
from urllib.request import urlopen
import matplotlib.pyplot as plt
import seaborn as sns
warnings.filterwarnings('ignore')
# %matplotlib inline
import os
import json
import re

data_path = "/Users/kaushik/Desktop/runasdus/src/com/lab/data/"

for fname in glob.glob(r"{}/*".format(data_path)):
    pol = os.path.join(fname, "priv.html")
    try:
        html = open(pol, "r").read()
        soup = BeautifulSoup(html, features="html.parser")
        soup_text = soup.get_text()
        print(soup_text)
    except:
        print("Exception occurred!")
        continue

for dirname in os.listdir("/Users/kaushik/Desktop/runasdus/src/com/lab/data/"):
    pol = os.path.abspath(os.path.join(dirname, "dom_ind.html"))
    print(pol)
    try:
        html = open(pol, "r").read()
        soup = BeautifulSoup(html, features="html.parser")
        soup_text = soup.get_text()
        print(soup_text)
    except:
        continue
    

for fname in glob.glob(r"{}/*.csv".format(annot_dpath)):   
    
    #Extract path basename
    basename = os.path.basename(fname)

    #Create directories if they don't exist
    os.makedirs(op_annotations_dpath, exist_ok = True)
    os.makedirs(op_segments_dpath, exist_ok = True)
    
    #Extract policyID from basename
    policy_id = basename.split('_')[0]
    policy_df = pd.read_csv(fname, header=None, usecols=[0, 4, 5, 6], names=['annotation_ID', 'segment_ID', 'category', 'attr_val'])
    
    #Set policyID in each table
    policy_df.loc[:,"policy_ID"] = policy_id
    
    #Replace extension
    santized_policy_fpath = os.path.splitext(basename)[0]+'.html'
    
    # Parse html text
    html = open(os.path.join(sanitized_pol_dpath, santized_policy_fpath), "r").read()
    soup = BeautifulSoup(html, features="html.parser")
    soup_text = soup.get_text()

StarcoderdataJupyterScriptsDedupFiltered