# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import requests
import csv
from bs4 import BeautifulSoup
import time
import random
from lxml import etree

def fetchUrl(url):
    '''
    Func: fetchUrl is used to get html page content
    Parameter:
    url : the url for the website 
    Return : req.test as page content
    '''
    try:
        #imitate browser
        headers={'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
                'User_Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36',
                'Cookie':'JSESSIONID=12B519D32115B6B6E97D3250B32ED086',
                'Connection':'keep-alive',
                'Referrer':'www.google.com',
                'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
                'Cache-Control':'max-age=0'
                }
        #get page content 
        req = requests.get(url,headers = headers)
        # check request status for throw exception
        req.raise_for_status() 
        #encoding the content based on the response from the url
        req.encoding=req.apparent_encoding         
        #print(req.encoding)
        #print('Fetch URL successful.')
        return req.text
    except Exception as e:
        print(e) #throw error

        


        
def getDiseaseNameList(text,diseaseList):
    '''
    from search page get diseaseNameList
    '''
    try:
        html = etree.HTML(text)
        htmlText =etree.tostring(html,encoding='unicode')
        diseaseResult = html.xpath('//h3[@class="mr11 fl"]/a')
        for a in diseaseResult:
            diseaseDict = {}
            diseaseDict['DiseaseName'] = a.attrib.get('title')
            diseaseDict['DiseaseLink'] = a.attrib.get('href')
            #print(a.attrib.get('title'))
            #print(a.href)
            diseaseList.append(diseaseDict)
        return diseaseList
    except Exception as e:
        print(e)      
        
        
        
def getDiseaseSolution(diseaseList):
    '''
    from diseaseList with disease name and disease base url 
    get the solution for the disease
    '''   
    for disease in diseaseList:
        diseaseUrl = disease['DiseaseLink']+'zl'
        diseasePage = fetchUrl(diseaseUrl)
        print(diseaseUrl)
        disease['solution'] = parseDiseaseSolution(diseasePage)


def parseDiseaseSolution(diseasePage):
    
    '''
    from search page get diseaseNameList
    '''
    try:
        html = etree.HTML(diseasePage)
        htmlText = etree.tostring(html,encoding='unicode')
        diseaseSolution = html.xpath('//p[@class="spea"]')
        #print(diseaseSolution[0].text)
        
        return diseaseSolution[0].text
    except Exception as e:
        print(e)      
    
        
#def Spider():
'''
Func:调用函数
'''
urlbase = 'http://jb.9939.com/jbzz/xiaohuaneike/?page=%d'
pageNumber = 0
diseaseList = []

#获取消化内科病症名称及链接主体
for i in range(1,2):
    url = urlbase%i
    text = fetchUrl(url)
    print(url)
    getDiseaseNameList(text,diseaseList)

getDiseaseSolution(diseaseList)


#def main():
#    Spider()        
        
#if __name__ == '__main__':
#    main()
# -

getDiseaseSolution(diseaseList)





StarcoderdataJupyterScriptsDedupFiltered