# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import csv 
import requests 
from xml.dom import minidom
from ContentDownloader import ContentDownloader
from tqdm import tqdm
from ipdb import set_trace
import pandas as pd
import numpy as np

from bs4 import BeautifulSoup
from readability import Document



def loadXML(url = 'https://www.salvageautosauction.com/sitemap.xml',name=''): 
  
    # url of rss feed 
    # creating HTTP response object from given url 
    resp = requests.get(url) 
  
    # saving the xml file 
    with open('csv/'+url.split('/')[-1], 'wb') as f: 
        f.write(resp.content) 
        
        
def parseXML(xmlfile): 
  

    xmldoc = minidom.parse(name)
    itemlist = xmldoc.getElementsByTagName('loc')
    print('URLs found: ',len(itemlist),'\dProcessing XML file ...')
    urls = []
    for item in tqdm(itemlist):
        url = item.firstChild.nodeValue
        if 'vehicle_detail' in url:
            urls.append(url)
    
    return urls 


def find_bid(page):
    try:
        soup = BeautifulSoup(page, 'html.parser')
        dom = soup.findAll("p", {"class": "text-center"})[1]
        #return dom.contents[0]
        #return ''.join([s for s in dom.contents[0].split() if s.isdigit()])
        return int(''.join([s for s in dom.contents[0] if s.isdigit()]))
    except:
        return np.nan
    
    
def format_data(df):
    res_df = pd.DataFrame()
    for index, row in tqdm(df.iterrows()):
        pagetable = pd.read_html(row['Text'])[0]
        item_name = row['URL'].split('vehicle_detail/')[1].split('/')[0]

        pagetable.columns = ['Vehicle Name',item_name]
        pagetable = pagetable.set_index('Vehicle Name').T
        pagetable['bid'] = find_bid(page)

        res_df = res_df.append(pagetable,sort=False)

    return res_df


# +
import csv 
import requests 
from xml.dom import minidom
from ContentDownloader import ContentDownloader
from tqdm import tqdm
from ipdb import set_trace
import pandas as pd
import numpy as np

from bs4 import BeautifulSoup
from readability import Document



def loadXML(url = 'https://www.salvageautosauction.com/sitemap.xml',name=''): 
  
    # url of rss feed 
    # creating HTTP response object from given url 
    resp = requests.get(url) 
  
    # saving the xml file 
    with open('csv/'+url.split('/')[-1], 'wb') as f: 
        f.write(resp.content) 
        
        
def parseXML(xmlfile): 
  

    xmldoc = minidom.parse(name)
    itemlist = xmldoc.getElementsByTagName('loc')
    print('URLs found: ',len(itemlist),'\dProcessing XML file ...')
    urls = []
    for item in tqdm(itemlist):
        url = item.firstChild.nodeValue
        if 'vehicle_detail' in url:
            urls.append(url)
    
    return urls 


def find_bid(page):
    try:
        soup = BeautifulSoup(page, 'html.parser')
        dom = soup.findAll("p", {"class": "text-center"})[1]
        #return dom.contents[0]
        #return ''.join([s for s in dom.contents[0].split() if s.isdigit()])
        return int(''.join([s for s in dom.contents[0] if s.isdigit()]))
    except:
        return np.nan
    
    
def format_data(df):
    res_df = pd.DataFrame()
    for index, row in tqdm(df.iterrows()):
        pagetable = pd.read_html(row['Text'])[0]
        item_name = row['URL'].split('vehicle_detail/')[1].split('/')[0]

        pagetable.columns = ['Vehicle Name',item_name]
        pagetable = pagetable.set_index('Vehicle Name').T
        pagetable['bid'] = find_bid(page)

        res_df = res_df.append(pagetable,sort=False)
        res_df.to_csv('csv/formated_date.csv')
    return res_df


# +
xlm_name = 'SAA_sitemap.xml'
text_file_name = 'SalvageAutos.csv'
loadXML(name=xlm_name)
urls = parseXML(xlm_name)

scraped_data = ContentDownloader.run_url_download(batch_size=100,urls_list=urls[:300] ,path_to_csv=text_file_name)

#scraped_data = pd.read_csv('csv/SalvageAutos.csv')

res_df = format_data(scraped_data)
# -

res_df.to_csv('csv/formatted_date.csv')



StarcoderdataJupyterScriptsDedupFiltered