<jupyter_start><jupyter_text>셀레니움을 이용한 네이버 블로그 크롤러<jupyter_code>import platform
print(platform.architecture())
!python --version
pwd
# 네이버에서 검색어 입력받아 검색 한 후 블로그 메뉴를 선택하고
# 오른쪽에 있는 검색옵션 버튼을 눌러서
# 정렬 방식과 기간을 입력하기

#Step 0. 필요한 모듈과 라이브러리를 로딩하고 검색어를 입력 받습니다.
import sys
import os
import pandas as pd
import numpy as np
import math

from bs4 import BeautifulSoup
import requests
import urllib.request as req

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import tqdm
from tqdm.notebook import tqdm
query_txt = '성심당여행대전'
start_date= "20190101"
end_date= "20210501"
os.getenv('HOME')
webdriver.__version__
#Step 1. 크롬 웹브라우저 실행
path = os.getenv('HOME')+ '/chromedriver'
driver = webdriver.Chrome(path)
# 사이트 주소는 네이버
driver.get('https://blog.naver.com/')
time.sleep(1)
#Step 2. 네이버 검색창에 "검색어" 검색
element = driver.find_element_by_name("sectionBlogQuery").send_keys(query_txt)
# element.submit()
time.sleep(2)
driver.find_element_by_class_name("button.button_blog").click() # 관련도순 xpath
# element.find_element_by_css_selector("#header > div.header_common > div > div.area_search > form > fieldset > a.button.button_blog").click() # 관련도순 xpath
# element.clear()
# element.send_keys(query_txt)  # query_txt는 위에서 입력한 '이재용'
# element.submit()
#Step 1. 크롬 웹브라우저 실행
path = os.getenv('HOME')+ '/chromedriver'

driver = webdriver.Chrome(path)
# 사이트 주소는 네이버
driver.get('https://blog.naver.com/')
time.sleep(0.1)

#Step 2. 네이버 검색창에 "검색어" 검색
element = driver.find_element_by_name("sectionBlogQuery").send_keys(query_txt)
# element.submit()
time.sleep(0.1)

driver.find_element_by_class_name("button.button_blog").click() # 관련도순 xpath
time.sleep(0.1)


#Step 4. 오른쪽의 검색 옵션 버튼 클릭
driver.find_element_by_class_name("present_selected").click()


#Step 6. 날짜 입력
driver.find_element_by_id("search_start_date").send_keys(start_date)
driver.find_element_by_id("search_end_date").send_keys(end_date)
time.sleep(0.1)

driver.find_element_by_id("periodSearch").click()
time.sleep(0.1)

searched_post_num = driver.find_element_by_class_name('search_number').text
print(searched_post_num)

url_list = []
title_list = []

total_page = 600 
# total_page = math.ceil(int(searched_post_num.replace(',', '').strip('건')) / 7)
print('total_page :', total_page)

for i in tqdm(range(0, total_page)):  # 페이지 번호
    url = f'https://section.blog.naver.com/Search/Post.naver?pageNo={i}&rangeType=sim&orderBy=recentdate&startDate={start_date}&endDate={end_date}&keyword={query_txt}'
    driver.get(url)
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    print(soup)
#     time.sleep(0.5)
#     area = soup.findAll('div', {'class' : 'list_search_post'}) #.find_all('a', {'class' : 'url'})
#     print(area)
    
    # URL 크롤링 시작
#     titles = "a.sh_blog_title._sp_each_url._sp_each_title" # #content
# #     article_raw = driver.find_elements_by_class_name('area_list_search')
# #     article_raw = driver.find_elements_by_css_selector('#content > section > div.area_list_search > div:nth-child(1)')
    
#     article_raw = driver.find_elements_by_xpath(f'//*[@id="content"]/section/div[2]/div[{i}]')
    
#     print(article_raw)

#     # url 크롤링 시작    # 7개 
#     for article in article_raw:
#         url = article.get_attribute('href')   
#         print(url)
#         url_list.append(url)
    
#     # 제목 크롤링 시작    
#     for article in article_raw:
#         title = article.get_attribute('title')   
#         title_list.append(title)
    
#         print(title)
    
# print('url갯수: ', len(url_list))
# print('url갯수: ', len(title_list))

# df = pd.DataFrame({'url':url_list, 'title':title_list})

# # 저장하기
# df.to_csv("./blog_url.csv")
li = [2, 3, 4, 4, 5, 6, 7, 8]
len(li)
for i in range(0, 8, 2):
    print(i)
new = []
for i in range(0, len(li)-1, 2):
    new.append([li[i], li[i+1]])
new
article_raw = driver.find_elements_by_xpath('//*[@id="content"]/section/div[2]/div[1]')
# article_raw.get_attribute('href')
for i in article_raw:
    print(i.get_attribute('href'))
//*[@id="content"]/section/div[2]
//*[@id="content"]/section/div[2]
//*[@id="content"]/section/div[2]

//*[@id="content"]/section/div[2]/div[1]
//*[@id="content"]/section/div[2]/div[2]
//*[@id="content"]/section/div[2]/div[3]
...
//*[@id="content"]/section/div[2]/div[7]<jupyter_output><empty_output><jupyter_text>1 page = 7 posts72 page searchsample = https://section.blog.naver.com/Search/Post.naver?pageNo=1&rangeType=PERIOD&orderBy=sim&startDate=2019-01-01&endDate=2021-05-01&keyword=%EC%84%B1%EC%8B%AC%EB%8B%B9%EC%97%AC%ED%96%89%EB%8C%80%EC%A0%84<jupyter_code>## 제목 눌러서 블로그 페이지 열기
driver.find_element_by_class_name('title').click()
time.sleep(1)
type(searched_post_num), searched_post_num
import re
re.sub('^[0-9]', '', searched_post_num)
searched_post_num
searched_post_num.replace(',', '').replace('건', '')
total_page = math.ceil(int(searched_post_num.replace(',', '').strip('건')) / 7)
total_page<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup